Completely remove MetaElement

This dramatically affects the load process and removes one
hoop we had to jump through, which is the creation of the extra
intermediate MetaElement objects.

This allows us to more easily carry state discovered by the Loader
over to the Element constructor, as we need not add additional state
to the intermediate MetaElement for this. Instead we have the Element
initializer understand the LoadElement directly.

Summary of changes:

  * _loader/metaelement.py: Removed

  * _loader/loadelement.py: Added some attributes previously required on
    MetaElement

  * _loader/loader.py: Removed _collect_element() and collect_element_no_deps(),
    removing the process of Loader.load() which translates LoadElements into
    MetaElements completely.

  * _loader/init.py: Export LoadElement, Dependency and Symbol types, stop
    exporting MetaElement

  * _loader/metasource.py: Now take the 'first_pass' parameter as an argument

  * _artifactelement.py: Use a virtual LoadElement instead of a virtual
    MetaElement to instantiate the ArtifactElement objects.

  * _pluginfactory/elementfactory.py: Adjust to now take a LoadElement

  * _project.py: Adjust Project.create_element() to now take a LoadElement,
    and call the new Element._new_from_load_element() instead of the
    old Element._new_from_meta() function

  * element.py:

    - Now export Element._new_from_load_element() instead of Element._new_from_meta()

    - Adjust the constructor to do the LoadElement toplevel node parsing instead
      of expecting members on the MetaElement object

    - Added __load_sources() which parses out and creates MetaSource objects
      for the sake of instantiating the element's Source objects. Consequently
      this simplifies the scenario where workspaces are involved.

  * source.py: Adjusted to use the new `first_pass` parameter to MetaSource when
    creating a duplicate clone.

10 files changed, 204 insertions, 323 deletions

diff --git a/src/buildstream/_artifactelement.py b/src/buildstream/_artifactelement.py
index 4066cef06..44e52ea90 100644
--- a/src/buildstream/_artifactelement.py
+++ b/src/buildstream/_artifactelement.py
@@ -22,7 +22,8 @@ from typing import TYPE_CHECKING
 from . import Element
 from . import _cachekey
 from ._exceptions import ArtifactElementError
-from ._loader.metaelement import MetaElement
+from ._loader import LoadElement
+from .node import Node
 from .types import Scope
 
 if TYPE_CHECKING:
@@ -49,10 +50,9 @@ class ArtifactElement(Element):
         self._key = key
 
         project = context.get_toplevel_project()
-        meta = MetaElement(project, element)  # NOTE element has no .bst suffix
-        plugin_conf = None
+        load_element = LoadElement(Node.from_dict({}), element, project.loader)  # NOTE element has no .bst suffix
 
-        super().__init__(context, project, meta, plugin_conf)
+        super().__init__(context, project, load_element, None)
 
     # _new_from_artifact_ref():
     #
diff --git a/src/buildstream/_loader/__init__.py b/src/buildstream/_loader/__init__.py
index fd5cac2ae..a4be9cfe5 100644
--- a/src/buildstream/_loader/__init__.py
+++ b/src/buildstream/_loader/__init__.py
@@ -17,7 +17,8 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
+from .types import Symbol
 from .metasource import MetaSource
-from .metaelement import MetaElement
+from .loadelement import LoadElement, Dependency
 from .loadcontext import LoadContext
 from .loader import Loader
diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx
index 6cd5b46b7..01334d124 100644
--- a/src/buildstream/_loader/loadelement.pyx
+++ b/src/buildstream/_loader/loadelement.pyx
@@ -184,9 +184,10 @@ cdef class LoadElement:
 
     cdef readonly MappingNode node
     cdef readonly str name
-    cdef readonly full_name
-    cdef public bint meta_done
+    cdef readonly str full_name
+    cdef readonly str kind
     cdef int node_id
+    cdef readonly bint first_pass
     cdef readonly object _loader
     cdef readonly str link_target
     cdef readonly ProvenanceInformation link_target_provenance
@@ -199,10 +200,10 @@ cdef class LoadElement:
         #
         # Public members
         #
+        self.kind = None        # The Element kind
         self.node = node        # The YAML node
         self.name = filename    # The element name
         self.full_name = None   # The element full name (with associated junction)
-        self.meta_done = False  # If the MetaElement for this LoadElement is done
         self.node_id = _next_synthetic_counter()
         self.link_target = None  # The target of a link element
         self.link_target_provenance = None  # The provenance of the link target
@@ -233,13 +234,15 @@ cdef class LoadElement:
             'build-depends', 'runtime-depends',
         ])
 
+        self.kind = node.get_str(Symbol.KIND, default=None)
+        self.first_pass = self.kind in ("junction", "link")
+
         #
         # If this is a link, resolve it right away and just
         # store the link target and provenance
         #
-        if self.node.get_str(Symbol.KIND, default=None) == 'link':
-            meta_element = self._loader.collect_element_no_deps(self)
-            element = Element._new_from_meta(meta_element)
+        if self.kind == 'link':
+            element = Element._new_from_load_element(self)
             element._initialize_state()
 
             # Custom error for link dependencies, since we don't completely
@@ -254,6 +257,36 @@ cdef class LoadElement:
             self.link_target = element.target
             self.link_target_provenance = element.target_provenance
 
+        # We don't count progress for junction elements or link
+        # as they do not represent real elements in the build graph.
+        #
+        # We check for a `None` kind, to avoid reporting progress for
+        # the virtual toplevel element used to load the pipeline.
+        #
+        if self._loader.load_context.task and self.kind is not None and not self.first_pass:
+            self._loader.load_context.task.add_current_progress()
+
+    # provenance
+    #
+    # A property reporting the ProvenanceInformation of the element
+    #
+    @property
+    def provenance(self):
+        return self.node.get_provenance()
+
+    # project
+    #
+    # A property reporting the Project in which this element resides.
+    #
+    @property
+    def project(self):
+        return self._loader.project
+
+    # junction
+    #
+    # A property reporting the junction element accessing this
+    # element, if any.
+    #
     @property
     def junction(self):
         return self._loader.project.junction
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index b0f4a4a07..94ee9078b 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -32,8 +32,6 @@ from ._loader import valid_chars_name
 from .types import Symbol
 from . import loadelement
 from .loadelement import LoadElement, Dependency, extract_depends_from_node
-from .metaelement import MetaElement
-from .metasource import MetaSource
 from ..types import CoreWarnings, _KeyStrength
 from .._message import Message, MessageType
 
@@ -41,8 +39,8 @@ from .._message import Message, MessageType
 # Loader():
 #
 # The Loader class does the heavy lifting of parsing target
-# bst files and ultimately transforming them into a list of MetaElements
-# with their own MetaSources, ready for instantiation by the core.
+# bst files and ultimately transforming them into a list of LoadElements
+# ready for instantiation by the core.
 #
 # Args:
 #    project (Project): The toplevel Project object
@@ -112,7 +110,8 @@ class Loader:
     #
     # Raises: LoadError
     #
-    # Returns: The toplevel LoadElement
+    # Returns:
+    #    (list): The corresponding LoadElement instances matching the `targets`
     #
     def load(self, targets):
 
@@ -154,7 +153,6 @@ class Loader:
         with PROFILER.profile(Topics.CIRCULAR_CHECK, "_".join(targets)):
             self._check_circular_deps(dummy_target)
 
-        ret = []
         #
         # Sort direct dependencies of elements by their dependency ordering
         #
@@ -167,18 +165,13 @@ class Loader:
             with PROFILER.profile(Topics.SORT_DEPENDENCIES, element.name):
                 loadelement.sort_dependencies(element, visited_elements)
 
-            # Finally, wrap what we have into LoadElements and return the target
-            #
-            ret.append(loader._collect_element(element))
-
         self._clean_caches()
 
         # Cache how many Elements have just been loaded
         if self.load_context.task:
-            # Workaround for task potentially being None (because no State object)
             self.loaded = self.load_context.task.current_progress
 
-        return ret
+        return target_elements
 
     # get_loader():
     #
@@ -252,83 +245,6 @@ class Loader:
         for parent in self._alternative_parents:
             yield from foreach_parent(parent)
 
-    # collect_element_no_deps()
-    #
-    # Collect a single element, without its dependencies, into a meta_element
-    #
-    # NOTE: This is declared public in order to share with the LoadElement
-    #       and should not be used outside of that `_loader` module.
-    #
-    # Args:
-    #    element (LoadElement): The element for which to load a MetaElement
-    #    report_progress (bool): Whether to report progress for this element, this is
-    #                            because we ignore junctions and links when counting
-    #                            how many elements we load.
-    #
-    # Returns:
-    #    (MetaElement): A partially loaded MetaElement
-    #
-    def collect_element_no_deps(self, element, *, report_progress=False):
-        # Return the already built one, if we already built it
-        meta_element = self._meta_elements.get(element.name)
-        if meta_element:
-            return meta_element
-
-        node = element.node
-        elt_provenance = node.get_provenance()
-        meta_sources = []
-
-        element_kind = node.get_str(Symbol.KIND)
-
-        # if there's a workspace for this element then just append a dummy workspace
-        # metasource.
-        workspace = self.load_context.context.get_workspaces().get_workspace(element.full_name)
-        skip_workspace = True
-        if workspace:
-            workspace_node = {"kind": "workspace"}
-            workspace_node["path"] = workspace.get_absolute_path()
-            workspace_node["last_build"] = str(workspace.to_dict().get("last_build", ""))
-            node[Symbol.SOURCES] = [workspace_node]
-            skip_workspace = False
-
-        sources = node.get_sequence(Symbol.SOURCES, default=[])
-        for index, source in enumerate(sources):
-            kind = source.get_str(Symbol.KIND)
-            # the workspace source plugin cannot be used unless the element is workspaced
-            if kind == "workspace" and skip_workspace:
-                continue
-
-            del source[Symbol.KIND]
-
-            # Directory is optional
-            directory = source.get_str(Symbol.DIRECTORY, default=None)
-            if directory:
-                del source[Symbol.DIRECTORY]
-            meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
-            meta_sources.append(meta_source)
-
-        meta_element = MetaElement(
-            self.project,
-            element.name,
-            element_kind,
-            elt_provenance,
-            meta_sources,
-            node.get_mapping(Symbol.CONFIG, default={}),
-            node.get_mapping(Symbol.VARIABLES, default={}),
-            node.get_mapping(Symbol.ENVIRONMENT, default={}),
-            node.get_str_list(Symbol.ENV_NOCACHE, default=[]),
-            node.get_mapping(Symbol.PUBLIC, default={}),
-            node.get_mapping(Symbol.SANDBOX, default={}),
-            element_kind in ("junction", "link"),
-        )
-
-        # Cache it now, make sure it's already there before recursing
-        self._meta_elements[element.name] = meta_element
-        if self.load_context.task and report_progress:
-            self.load_context.task.add_current_progress()
-
-        return meta_element
-
     ###########################################
     #            Private Methods              #
     ###########################################
@@ -553,52 +469,6 @@ class Loader:
                 check_elements.remove(this_element)
                 validated.add(this_element)
 
-    # _collect_element()
-    #
-    # Collect the toplevel elements we have
-    #
-    # Args:
-    #    top_element (LoadElement): The element for which to load a MetaElement
-    #
-    # Returns:
-    #    (MetaElement): A fully loaded MetaElement
-    #
-    def _collect_element(self, top_element):
-        element_queue = [top_element]
-        meta_element_queue = [self.collect_element_no_deps(top_element, report_progress=True)]
-
-        while element_queue:
-            element = element_queue.pop()
-            meta_element = meta_element_queue.pop()
-
-            if element.meta_done:
-                # This can happen if there are multiple top level targets
-                # in which case, we simply skip over this element.
-                continue
-
-            for dep in element.dependencies:
-
-                loader = dep.element._loader
-                name = dep.element.name
-
-                try:
-                    meta_dep = loader._meta_elements[name]
-                except KeyError:
-                    meta_dep = loader.collect_element_no_deps(dep.element, report_progress=True)
-                    element_queue.append(dep.element)
-                    meta_element_queue.append(meta_dep)
-
-                if dep.dep_type != "runtime":
-                    meta_element.build_dependencies.append(meta_dep)
-                if dep.dep_type != "build":
-                    meta_element.dependencies.append(meta_dep)
-                if dep.strict:
-                    meta_element.strict_dependencies.append(meta_dep)
-
-            element.meta_done = True
-
-        return self._meta_elements[top_element.name]
-
     # _search_for_override():
     #
     # Search parent projects for an overridden subproject to replace this junction.
@@ -711,20 +581,9 @@ class Loader:
         if not load_subprojects:
             return None
 
-        # meta junction element
-        #
-        # Note that junction elements are not allowed to have
-        # dependencies, so disabling progress reporting here should
-        # have no adverse effects - the junction element itself cannot
-        # be depended on, so it would be confusing for its load to
-        # show up in logs.
-        #
-        # Any task counting *inside* the junction will be handled by
-        # its loader.
-        meta_element = self.collect_element_no_deps(self._elements[filename])
-        if meta_element.kind != "junction":
+        if load_element.kind != "junction":
             raise LoadError(
-                "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind),
+                "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, load_element.kind),
                 LoadErrorReason.INVALID_DATA,
             )
 
@@ -748,7 +607,7 @@ class Loader:
                 "{}: Dependencies are forbidden for 'junction' elements".format(p), LoadErrorReason.INVALID_JUNCTION
             )
 
-        element = Element._new_from_meta(meta_element)
+        element = Element._new_from_load_element(load_element)
         element._initialize_state()
 
         # Handle the case where a subproject has no ref
diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py
deleted file mode 100644
index 1c1f6feb2..000000000
--- a/src/buildstream/_loader/metaelement.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-#  Copyright (C) 2016 Codethink Limited
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 2 of the License, or (at your option) any later version.
-#
-#  This library is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
-#  Lesser General Public License for more details.
-#
-#  You should have received a copy of the GNU Lesser General Public
-#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-#  Authors:
-#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
-
-from ..node import Node
-
-
-class MetaElement:
-
-    # MetaElement()
-    #
-    # An abstract object holding data suitable for constructing an Element
-    #
-    # Args:
-    #    project: The project that contains the element
-    #    name: The resolved element name
-    #    kind: The element kind
-    #    provenance: The provenance of the element
-    #    sources: An array of MetaSource objects
-    #    config: The configuration data for the element
-    #    variables: The variables declared or overridden on this element
-    #    environment: The environment variables declared or overridden on this element
-    #    env_nocache: List of environment vars which should not be considered in cache keys
-    #    public: Public domain data dictionary
-    #    sandbox: Configuration specific to the sandbox environment
-    #    first_pass: The element is to be loaded with first pass configuration (junction)
-    #
-    def __init__(
-        self,
-        project,
-        name,
-        kind=None,
-        provenance=None,
-        sources=None,
-        config=None,
-        variables=None,
-        environment=None,
-        env_nocache=None,
-        public=None,
-        sandbox=None,
-        first_pass=False,
-    ):
-        self.project = project
-        self.name = name
-        self.kind = kind
-        self.provenance = provenance
-        self.sources = sources
-        self.config = config or Node.from_dict({})
-        self.variables = variables or Node.from_dict({})
-        self.environment = environment or Node.from_dict({})
-        self.env_nocache = env_nocache or []
-        self.public = public or Node.from_dict({})
-        self.sandbox = sandbox or Node.from_dict({})
-        self.build_dependencies = []
-        self.dependencies = []
-        self.strict_dependencies = []
-        self.first_pass = first_pass
-        self.is_junction = kind in ("junction", "link")
diff --git a/src/buildstream/_loader/metasource.py b/src/buildstream/_loader/metasource.py
index ddaa538f5..d49ae3d82 100644
--- a/src/buildstream/_loader/metasource.py
+++ b/src/buildstream/_loader/metasource.py
@@ -32,12 +32,12 @@ class MetaSource:
     #    config: The configuration data for the source
     #    first_pass: This source will be used with first project pass configuration (used for junctions).
     #
-    def __init__(self, element_name, element_index, element_kind, kind, config, directory):
+    def __init__(self, element_name, element_index, element_kind, kind, config, directory, first_pass):
         self.element_name = element_name
         self.element_index = element_index
         self.element_kind = element_kind
         self.kind = kind
         self.config = config
         self.directory = directory
-        self.first_pass = False
+        self.first_pass = first_pass
         self.provenance = config.get_provenance()
diff --git a/src/buildstream/_pluginfactory/elementfactory.py b/src/buildstream/_pluginfactory/elementfactory.py
index 9854c1a5c..7d94c2541 100644
--- a/src/buildstream/_pluginfactory/elementfactory.py
+++ b/src/buildstream/_pluginfactory/elementfactory.py
@@ -39,7 +39,7 @@ class ElementFactory(PluginFactory):
     # Args:
     #    context (object): The Context object for processing
     #    project (object): The project object
-    #    meta (object): The loaded MetaElement
+    #    load_element (object): The LoadElement
     #
     # Returns: A newly created Element object of the appropriate kind
     #
@@ -47,7 +47,7 @@ class ElementFactory(PluginFactory):
     #    PluginError (if the kind lookup failed)
     #    LoadError (if the element itself took issue with the config)
     #
-    def create(self, context, project, meta):
-        element_type, default_config = self.lookup(context.messenger, meta.kind, meta.provenance)
-        element = element_type(context, project, meta, default_config)
+    def create(self, context, project, load_element):
+        element_type, default_config = self.lookup(context.messenger, load_element.kind, load_element.provenance)
+        element = element_type(context, project, load_element, default_config)
         return element
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 3562ea5d4..87772995a 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -345,13 +345,13 @@ class Project:
     # Instantiate and return an element
     #
     # Args:
-    #    meta (MetaElement): The loaded MetaElement
+    #    load_element (LoadElement): The LoadElement
     #
     # Returns:
     #    (Element): A newly created Element object of the appropriate kind
     #
-    def create_element(self, meta):
-        return self.element_factory.create(self._context, self, meta)
+    def create_element(self, load_element):
+        return self.element_factory.create(self._context, self, load_element)
 
     # create_source()
     #
@@ -426,13 +426,13 @@ class Project:
 
         with self._context.messenger.simple_task("Loading elements", silent_nested=True) as task:
             self.load_context.set_task(task)
-            meta_elements = self.loader.load(targets)
+            load_elements = self.loader.load(targets)
             self.load_context.set_task(None)
 
         with self._context.messenger.simple_task("Resolving elements") as task:
             if task:
                 task.set_maximum_progress(self.loader.loaded)
-            elements = [Element._new_from_meta(meta, task) for meta in meta_elements]
+            elements = [Element._new_from_load_element(load_element, task) for load_element in load_elements]
 
         Element._clear_meta_elements_cache()
 
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index f1e909bce..f7d49d57f 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -80,7 +80,7 @@ import copy
 import warnings
 from collections import OrderedDict
 import contextlib
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from functools import partial
 from itertools import chain
 import string
@@ -106,6 +106,7 @@ from .sandbox._sandboxremote import SandboxRemote
 from .types import CoreWarnings, Scope, _CacheBuildTrees, _KeyStrength
 from ._artifact import Artifact
 from ._elementsources import ElementSources
+from ._loader import Symbol, MetaSource
 
 from .storage.directory import Directory
 from .storage._filebaseddirectory import FileBasedDirectory
@@ -120,7 +121,7 @@ if TYPE_CHECKING:
     from .sandbox import Sandbox
     from .source import Source
     from ._context import Context
-    from ._loader.metaelement import MetaElement
+    from ._loader import LoadElement
     from ._project import Project
 
     # pylint: enable=cyclic-import
@@ -157,8 +158,8 @@ class Element(Plugin):
 
     # The defaults from the yaml file and project
     __defaults = None
-    # A hash of Element by MetaElement
-    __instantiated_elements = {}  # type: Dict[MetaElement, Element]
+    # A hash of Element by LoadElement
+    __instantiated_elements = {}  # type: Dict[LoadElement, Element]
     # A list of (source, ref) tuples which were redundantly specified
     __redundant_source_refs = []  # type: List[Tuple[Source, SourceRef]]
 
@@ -197,15 +198,17 @@ class Element(Plugin):
     """Whether the element produces an artifact when built.
     """
 
-    def __init__(self, context: "Context", project: "Project", meta: "MetaElement", plugin_conf: Dict[str, Any]):
+    def __init__(
+        self, context: "Context", project: "Project", load_element: "LoadElement", plugin_conf: Dict[str, Any]
+    ):
 
         self.__cache_key_dict = None  # Dict for cache key calculation
         self.__cache_key = None  # Our cached cache key
 
-        super().__init__(meta.name, context, project, meta.provenance, "element")
+        super().__init__(load_element.name, context, project, load_element.provenance, "element")
 
         # Ensure the project is fully loaded here rather than later on
-        if not meta.is_junction:
+        if not load_element.first_pass:
             project.ensure_fully_loaded()
 
         self.project_name = self._get_project().name
@@ -270,44 +273,44 @@ class Element(Plugin):
         self.__resolved_initial_state = False  # Whether the initial state of the Element has been resolved
 
         # Ensure we have loaded this class's defaults
-        self.__init_defaults(project, plugin_conf, meta.kind, meta.is_junction)
+        self.__init_defaults(project, plugin_conf, load_element.kind, load_element.first_pass)
 
         # Collect the composited variables and resolve them
-        variables = self.__extract_variables(project, meta)
+        variables = self.__extract_variables(project, load_element)
         variables["element-name"] = self.name
         self.__variables = Variables(variables)
-        if not meta.is_junction:
+        if not load_element.first_pass:
             self.__variables.check()
 
         # Collect the composited environment now that we have variables
-        unexpanded_env = self.__extract_environment(project, meta)
+        unexpanded_env = self.__extract_environment(project, load_element)
         self.__variables.expand(unexpanded_env)
         self.__environment = unexpanded_env.strip_node_info()
 
         # Collect the environment nocache blacklist list
-        nocache = self.__extract_env_nocache(project, meta)
+        nocache = self.__extract_env_nocache(project, load_element)
         self.__env_nocache = nocache
 
         # Grab public domain data declared for this instance
-        self.__public = self.__extract_public(meta)
+        self.__public = self.__extract_public(load_element)
         self.__variables.expand(self.__public)
         self.__dynamic_public = None
 
         # Collect the composited element configuration and
         # ask the element to configure itself.
-        self.__config = self.__extract_config(meta)
+        self.__config = self.__extract_config(load_element)
         self.__variables.expand(self.__config)
 
         self._configure(self.__config)
 
         # Extract remote execution URL
-        if meta.is_junction:
+        if load_element.first_pass:
             self.__remote_execution_specs = None
         else:
             self.__remote_execution_specs = project.remote_execution_specs
 
         # Extract Sandbox config
-        sandbox_config = self.__extract_sandbox_config(project, meta)
+        sandbox_config = self.__extract_sandbox_config(project, load_element)
         self.__variables.expand(sandbox_config)
         self.__sandbox_config = SandboxConfig(sandbox_config, context.platform)
 
@@ -874,61 +877,56 @@ class Element(Plugin):
     #            Private Methods used in BuildStream            #
     #############################################################
 
-    # _new_from_meta():
+    # _new_from_load_element():
     #
     # Recursively instantiate a new Element instance, its sources
-    # and its dependencies from a meta element.
+    # and its dependencies from a LoadElement.
+    #
+    # FIXME: Need to use an iterative algorithm here since recursion
+    #        will limit project dependency depth.
     #
     # Args:
-    #    meta (MetaElement): The meta element
+    #    load_element (LoadElement): The LoadElement
     #    task (Task): A task object to report progress to
     #
     # Returns:
     #    (Element): A newly created Element instance
     #
     @classmethod
-    def _new_from_meta(cls, meta, task=None):
+    def _new_from_load_element(cls, load_element, task=None):
 
-        if not meta.first_pass:
-            meta.project.ensure_fully_loaded()
+        if not load_element.first_pass:
+            load_element.project.ensure_fully_loaded()
 
-        if meta in cls.__instantiated_elements:
-            return cls.__instantiated_elements[meta]
+        with suppress(KeyError):
+            return cls.__instantiated_elements[load_element]
 
-        element = meta.project.create_element(meta)
-        cls.__instantiated_elements[meta] = element
+        element = load_element.project.create_element(load_element)
+        cls.__instantiated_elements[load_element] = element
 
-        # Instantiate sources and generate their keys
-        for meta_source in meta.sources:
-            meta_source.first_pass = meta.is_junction
-            source = meta.project.create_source(meta_source, variables=element.__variables)
+        # Load the sources from the LoadElement
+        element.__load_sources(load_element)
 
-            redundant_ref = source._load_ref()
+        # Instantiate dependencies
+        for dep in load_element.dependencies:
+            dependency = Element._new_from_load_element(dep.element, task)
 
-            element.__sources.add_source(source)
+            if dep.dep_type != "runtime":
+                element.__build_dependencies.append(dependency)
+                dependency.__reverse_build_deps.add(element)
 
-            # Collect redundant refs which occurred at load time
-            if redundant_ref is not None:
-                cls.__redundant_source_refs.append((source, redundant_ref))
+            if dep.dep_type != "build":
+                element.__runtime_dependencies.append(dependency)
+                dependency.__reverse_runtime_deps.add(element)
+
+            if dep.strict:
+                element.__strict_dependencies.append(dependency)
 
-        # Instantiate dependencies
-        for meta_dep in meta.dependencies:
-            dependency = Element._new_from_meta(meta_dep, task)
-            element.__runtime_dependencies.append(dependency)
-            dependency.__reverse_runtime_deps.add(element)
         no_of_runtime_deps = len(element.__runtime_dependencies)
         element.__runtime_deps_without_strict_cache_key = no_of_runtime_deps
         element.__runtime_deps_without_cache_key = no_of_runtime_deps
         element.__runtime_deps_uncached = no_of_runtime_deps
 
-        for meta_dep in meta.build_dependencies:
-            dependency = Element._new_from_meta(meta_dep, task)
-            element.__build_dependencies.append(dependency)
-            dependency.__reverse_build_deps.add(element)
-
-            if meta_dep in meta.strict_dependencies:
-                element.__strict_dependencies.append(dependency)
-
         no_of_build_deps = len(element.__build_dependencies)
         element.__build_deps_without_strict_cache_key = no_of_build_deps
         element.__build_deps_without_cache_key = no_of_build_deps
@@ -2203,6 +2201,63 @@ class Element(Plugin):
     #                   Private Local Methods                   #
     #############################################################
 
+    # __load_sources()
+    #
+    # Load the Source objects from the LoadElement
+    #
+    def __load_sources(self, load_element):
+        project = self._get_project()
+        workspace = self._get_workspace()
+        meta_sources = []
+
+        # If there's a workspace for this element then we just load a workspace
+        # source plugin instead of the real plugins
+        if workspace:
+            workspace_node = {"kind": "workspace"}
+            workspace_node["path"] = workspace.get_absolute_path()
+            workspace_node["last_build"] = str(workspace.to_dict().get("last_build", ""))
+            meta = MetaSource(
+                self.name,
+                0,
+                self.get_kind(),
+                "workspace",
+                Node.from_dict(workspace_node),
+                None,
+                load_element.first_pass,
+            )
+            meta_sources.append(meta)
+        else:
+            sources = load_element.node.get_sequence(Symbol.SOURCES, default=[])
+            for index, source in enumerate(sources):
+                kind = source.get_scalar(Symbol.KIND)
+
+                # The workspace source plugin is only valid for internal use
+                if kind.as_str() == "workspace":
+                    raise LoadError(
+                        "{}: Invalid usage of workspace source kind".format(kind.get_provenance()),
+                        LoadErrorReason.INVALID_DATA,
+                    )
+                del source[Symbol.KIND]
+
+                # Directory is optional
+                directory = source.get_str(Symbol.DIRECTORY, default=None)
+                if directory:
+                    del source[Symbol.DIRECTORY]
+                meta_source = MetaSource(
+                    self.name, index, self.get_kind(), kind.as_str(), source, directory, load_element.first_pass
+                )
+                meta_sources.append(meta_source)
+
+        for meta_source in meta_sources:
+            source = project.create_source(meta_source, variables=self.__variables)
+            redundant_ref = source._load_ref()
+
+            self.__sources.add_source(source)
+
+            # Collect redundant refs which occurred at load time
+            if redundant_ref is not None:
+                self.__redundant_source_refs.append((source, redundant_ref))
+
     # __get_dependency_refs()
     #
     # Retrieve the artifact refs of the element's dependencies
@@ -2433,13 +2488,13 @@ class Element(Plugin):
                 yield sandbox
 
     @classmethod
-    def __compose_default_splits(cls, project, defaults, is_junction):
+    def __compose_default_splits(cls, project, defaults, first_pass):
 
-        element_public = defaults.get_mapping("public", default={})
+        element_public = defaults.get_mapping(Symbol.PUBLIC, default={})
         element_bst = element_public.get_mapping("bst", default={})
         element_splits = element_bst.get_mapping("split-rules", default={})
 
-        if is_junction:
+        if first_pass:
             splits = element_splits.clone()
         else:
             assert project._splits is not None
@@ -2450,10 +2505,10 @@ class Element(Plugin):
 
         element_bst["split-rules"] = splits
         element_public["bst"] = element_bst
-        defaults["public"] = element_public
+        defaults[Symbol.PUBLIC] = element_public
 
     @classmethod
-    def __init_defaults(cls, project, plugin_conf, kind, is_junction):
+    def __init_defaults(cls, project, plugin_conf, kind, first_pass):
         # Defaults are loaded once per class and then reused
         #
         if cls.__defaults is None:
@@ -2468,11 +2523,11 @@ class Element(Plugin):
                         raise e
 
             # Special case; compose any element-wide split-rules declarations
-            cls.__compose_default_splits(project, defaults, is_junction)
+            cls.__compose_default_splits(project, defaults, first_pass)
 
             # Override the element's defaults with element specific
             # overrides from the project.conf
-            if is_junction:
+            if first_pass:
                 elements = project.first_pass_config.element_overrides
             else:
                 elements = project.element_overrides
@@ -2488,29 +2543,30 @@ class Element(Plugin):
     # creating sandboxes for this element
     #
     @classmethod
-    def __extract_environment(cls, project, meta):
-        default_env = cls.__defaults.get_mapping("environment", default={})
+    def __extract_environment(cls, project, load_element):
+        default_env = cls.__defaults.get_mapping(Symbol.ENVIRONMENT, default={})
+        element_env = load_element.node.get_mapping(Symbol.ENVIRONMENT, default={}) or Node.from_dict({})
 
-        if meta.is_junction:
+        if load_element.first_pass:
             environment = Node.from_dict({})
         else:
             environment = project.base_environment.clone()
 
         default_env._composite(environment)
-        meta.environment._composite(environment)
+        element_env._composite(environment)
         environment._assert_fully_composited()
 
         return environment
 
     @classmethod
-    def __extract_env_nocache(cls, project, meta):
-        if meta.is_junction:
+    def __extract_env_nocache(cls, project, load_element):
+        if load_element.first_pass:
             project_nocache = []
         else:
             project_nocache = project.base_env_nocache
 
-        default_nocache = cls.__defaults.get_str_list("environment-nocache", default=[])
-        element_nocache = meta.env_nocache
+        default_nocache = cls.__defaults.get_str_list(Symbol.ENV_NOCACHE, default=[])
+        element_nocache = load_element.node.get_str_list(Symbol.ENV_NOCACHE, default=[])
 
         # Accumulate values from the element default, the project and the element
         # itself to form a complete list of nocache env vars.
@@ -2523,16 +2579,17 @@ class Element(Plugin):
     # substituting command strings to be run in the sandbox
     #
     @classmethod
-    def __extract_variables(cls, project, meta):
-        default_vars = cls.__defaults.get_mapping("variables", default={})
+    def __extract_variables(cls, project, load_element):
+        default_vars = cls.__defaults.get_mapping(Symbol.VARIABLES, default={})
+        element_vars = load_element.node.get_mapping(Symbol.VARIABLES, default={}) or Node.from_dict({})
 
-        if meta.is_junction:
+        if load_element.first_pass:
             variables = project.first_pass_config.base_variables.clone()
         else:
             variables = project.base_variables.clone()
 
         default_vars._composite(variables)
-        meta.variables._composite(variables)
+        element_vars._composite(variables)
         variables._assert_fully_composited()
 
         for var in ("project-name", "element-name", "max-jobs"):
@@ -2554,13 +2611,14 @@ class Element(Plugin):
     # off to element.configure()
     #
     @classmethod
-    def __extract_config(cls, meta):
+    def __extract_config(cls, load_element):
+        element_config = load_element.node.get_mapping(Symbol.CONFIG, default={}) or Node.from_dict({})
 
         # The default config is already composited with the project overrides
-        config = cls.__defaults.get_mapping("config", default={})
+        config = cls.__defaults.get_mapping(Symbol.CONFIG, default={})
         config = config.clone()
 
-        meta.config._composite(config)
+        element_config._composite(config)
         config._assert_fully_composited()
 
         return config
@@ -2568,18 +2626,20 @@ class Element(Plugin):
     # Sandbox-specific configuration data, to be passed to the sandbox's constructor.
     #
     @classmethod
-    def __extract_sandbox_config(cls, project, meta):
-        if meta.is_junction:
+    def __extract_sandbox_config(cls, project, load_element):
+        element_sandbox = load_element.node.get_mapping(Symbol.SANDBOX, default={}) or Node.from_dict({})
+
+        if load_element.first_pass:
             sandbox_config = Node.from_dict({})
         else:
             sandbox_config = project._sandbox.clone()
 
         # The default config is already composited with the project overrides
-        sandbox_defaults = cls.__defaults.get_mapping("sandbox", default={})
+        sandbox_defaults = cls.__defaults.get_mapping(Symbol.SANDBOX, default={})
         sandbox_defaults = sandbox_defaults.clone()
 
         sandbox_defaults._composite(sandbox_config)
-        meta.sandbox._composite(sandbox_config)
+        element_sandbox._composite(sandbox_config)
         sandbox_config._assert_fully_composited()
 
         return sandbox_config
@@ -2588,14 +2648,16 @@ class Element(Plugin):
     # elements may extend but whos defaults are defined in the project.
     #
     @classmethod
-    def __extract_public(cls, meta):
-        base_public = cls.__defaults.get_mapping("public", default={})
+    def __extract_public(cls, load_element):
+        element_public = load_element.node.get_mapping(Symbol.PUBLIC, default={}) or Node.from_dict({})
+
+        base_public = cls.__defaults.get_mapping(Symbol.PUBLIC, default={})
         base_public = base_public.clone()
 
         base_bst = base_public.get_mapping("bst", default={})
         base_splits = base_bst.get_mapping("split-rules", default={})
 
-        element_public = meta.public.clone()
+        element_public = element_public.clone()
         element_bst = element_public.get_mapping("bst", default={})
         element_splits = element_bst.get_mapping("split-rules", default={})
 
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index dc87c35de..ea77a6537 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -1213,10 +1213,9 @@ class Source(Plugin):
             self.get_kind(),
             self.__config,
             self.__directory,
+            self.__first_pass,
         )
 
-        meta.first_pass = self.__first_pass
-
         clone = source_kind(
             context, project, meta, self.__variables, alias_override=(alias, uri), unique_id=self._unique_id
         )