Completely remove MetaElement

This dramatically affects the load process and removes one
hoop we had to jump through, which is the creation of the extra
intermediate MetaElement objects.

This allows us to more easily carry state discovered by the Loader
over to the Element constructor, as we need not add additional state
to the intermediate MetaElement for this. Instead we have the Element
initializer understand the LoadElement directly.

Summary of changes:

  * _loader/metaelement.py: Removed

  * _loader/loadelement.py: Added some attributes previously required on
    MetaElement

  * _loader/loader.py: Removed _collect_element() and collect_element_no_deps(),
    removing the process of Loader.load() which translates LoadElements into
    MetaElements completely.

  * _loader/init.py: Export LoadElement, Dependency and Symbol types, stop
    exporting MetaElement

  * _loader/metasource.py: Now take the 'first_pass' parameter as an argument

  * _artifactelement.py: Use a virtual LoadElement instead of a virtual
    MetaElement to instantiate the ArtifactElement objects.

  * _pluginfactory/elementfactory.py: Adjust to now take a LoadElement

  * _project.py: Adjust Project.create_element() to now take a LoadElement,
    and call the new Element._new_from_load_element() instead of the
    old Element._new_from_meta() function

  * element.py:

    - Now export Element._new_from_load_element() instead of Element._new_from_meta()

    - Adjust the constructor to do the LoadElement toplevel node parsing instead
      of expecting members on the MetaElement object

    - Added __load_sources() which parses out and creates MetaSource objects
      for the sake of instantiating the element's Source objects. Consequently
      this simplifies the scenario where workspaces are involved.

  * source.py: Adjusted to use the new `first_pass` parameter to MetaSource when
    creating a duplicate clone.

5 files changed, 51 insertions, 231 deletions

diff --git a/src/buildstream/_loader/__init__.py b/src/buildstream/_loader/__init__.py
index fd5cac2ae..a4be9cfe5 100644
--- a/src/buildstream/_loader/__init__.py
+++ b/src/buildstream/_loader/__init__.py
@@ -17,7 +17,8 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
+from .types import Symbol
 from .metasource import MetaSource
-from .metaelement import MetaElement
+from .loadelement import LoadElement, Dependency
 from .loadcontext import LoadContext
 from .loader import Loader
diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx
index 6cd5b46b7..01334d124 100644
--- a/src/buildstream/_loader/loadelement.pyx
+++ b/src/buildstream/_loader/loadelement.pyx
@@ -184,9 +184,10 @@ cdef class LoadElement:
 
     cdef readonly MappingNode node
     cdef readonly str name
-    cdef readonly full_name
-    cdef public bint meta_done
+    cdef readonly str full_name
+    cdef readonly str kind
     cdef int node_id
+    cdef readonly bint first_pass
     cdef readonly object _loader
     cdef readonly str link_target
     cdef readonly ProvenanceInformation link_target_provenance
@@ -199,10 +200,10 @@ cdef class LoadElement:
         #
         # Public members
         #
+        self.kind = None        # The Element kind
         self.node = node        # The YAML node
         self.name = filename    # The element name
         self.full_name = None   # The element full name (with associated junction)
-        self.meta_done = False  # If the MetaElement for this LoadElement is done
         self.node_id = _next_synthetic_counter()
         self.link_target = None  # The target of a link element
         self.link_target_provenance = None  # The provenance of the link target
@@ -233,13 +234,15 @@ cdef class LoadElement:
             'build-depends', 'runtime-depends',
         ])
 
+        self.kind = node.get_str(Symbol.KIND, default=None)
+        self.first_pass = self.kind in ("junction", "link")
+
         #
         # If this is a link, resolve it right away and just
         # store the link target and provenance
         #
-        if self.node.get_str(Symbol.KIND, default=None) == 'link':
-            meta_element = self._loader.collect_element_no_deps(self)
-            element = Element._new_from_meta(meta_element)
+        if self.kind == 'link':
+            element = Element._new_from_load_element(self)
             element._initialize_state()
 
             # Custom error for link dependencies, since we don't completely
@@ -254,6 +257,36 @@ cdef class LoadElement:
             self.link_target = element.target
             self.link_target_provenance = element.target_provenance
 
+        # We don't count progress for junction elements or link
+        # as they do not represent real elements in the build graph.
+        #
+        # We check for a `None` kind, to avoid reporting progress for
+        # the virtual toplevel element used to load the pipeline.
+        #
+        if self._loader.load_context.task and self.kind is not None and not self.first_pass:
+            self._loader.load_context.task.add_current_progress()
+
+    # provenance
+    #
+    # A property reporting the ProvenanceInformation of the element
+    #
+    @property
+    def provenance(self):
+        return self.node.get_provenance()
+
+    # project
+    #
+    # A property reporting the Project in which this element resides.
+    #
+    @property
+    def project(self):
+        return self._loader.project
+
+    # junction
+    #
+    # A property reporting the junction element accessing this
+    # element, if any.
+    #
     @property
     def junction(self):
         return self._loader.project.junction
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index b0f4a4a07..94ee9078b 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -32,8 +32,6 @@ from ._loader import valid_chars_name
 from .types import Symbol
 from . import loadelement
 from .loadelement import LoadElement, Dependency, extract_depends_from_node
-from .metaelement import MetaElement
-from .metasource import MetaSource
 from ..types import CoreWarnings, _KeyStrength
 from .._message import Message, MessageType
 
@@ -41,8 +39,8 @@ from .._message import Message, MessageType
 # Loader():
 #
 # The Loader class does the heavy lifting of parsing target
-# bst files and ultimately transforming them into a list of MetaElements
-# with their own MetaSources, ready for instantiation by the core.
+# bst files and ultimately transforming them into a list of LoadElements
+# ready for instantiation by the core.
 #
 # Args:
 #    project (Project): The toplevel Project object
@@ -112,7 +110,8 @@ class Loader:
     #
     # Raises: LoadError
     #
-    # Returns: The toplevel LoadElement
+    # Returns:
+    #    (list): The corresponding LoadElement instances matching the `targets`
     #
     def load(self, targets):
 
@@ -154,7 +153,6 @@ class Loader:
         with PROFILER.profile(Topics.CIRCULAR_CHECK, "_".join(targets)):
             self._check_circular_deps(dummy_target)
 
-        ret = []
         #
         # Sort direct dependencies of elements by their dependency ordering
         #
@@ -167,18 +165,13 @@ class Loader:
             with PROFILER.profile(Topics.SORT_DEPENDENCIES, element.name):
                 loadelement.sort_dependencies(element, visited_elements)
 
-            # Finally, wrap what we have into LoadElements and return the target
-            #
-            ret.append(loader._collect_element(element))
-
         self._clean_caches()
 
         # Cache how many Elements have just been loaded
         if self.load_context.task:
-            # Workaround for task potentially being None (because no State object)
             self.loaded = self.load_context.task.current_progress
 
-        return ret
+        return target_elements
 
     # get_loader():
     #
@@ -252,83 +245,6 @@ class Loader:
         for parent in self._alternative_parents:
             yield from foreach_parent(parent)
 
-    # collect_element_no_deps()
-    #
-    # Collect a single element, without its dependencies, into a meta_element
-    #
-    # NOTE: This is declared public in order to share with the LoadElement
-    #       and should not be used outside of that `_loader` module.
-    #
-    # Args:
-    #    element (LoadElement): The element for which to load a MetaElement
-    #    report_progress (bool): Whether to report progress for this element, this is
-    #                            because we ignore junctions and links when counting
-    #                            how many elements we load.
-    #
-    # Returns:
-    #    (MetaElement): A partially loaded MetaElement
-    #
-    def collect_element_no_deps(self, element, *, report_progress=False):
-        # Return the already built one, if we already built it
-        meta_element = self._meta_elements.get(element.name)
-        if meta_element:
-            return meta_element
-
-        node = element.node
-        elt_provenance = node.get_provenance()
-        meta_sources = []
-
-        element_kind = node.get_str(Symbol.KIND)
-
-        # if there's a workspace for this element then just append a dummy workspace
-        # metasource.
-        workspace = self.load_context.context.get_workspaces().get_workspace(element.full_name)
-        skip_workspace = True
-        if workspace:
-            workspace_node = {"kind": "workspace"}
-            workspace_node["path"] = workspace.get_absolute_path()
-            workspace_node["last_build"] = str(workspace.to_dict().get("last_build", ""))
-            node[Symbol.SOURCES] = [workspace_node]
-            skip_workspace = False
-
-        sources = node.get_sequence(Symbol.SOURCES, default=[])
-        for index, source in enumerate(sources):
-            kind = source.get_str(Symbol.KIND)
-            # the workspace source plugin cannot be used unless the element is workspaced
-            if kind == "workspace" and skip_workspace:
-                continue
-
-            del source[Symbol.KIND]
-
-            # Directory is optional
-            directory = source.get_str(Symbol.DIRECTORY, default=None)
-            if directory:
-                del source[Symbol.DIRECTORY]
-            meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
-            meta_sources.append(meta_source)
-
-        meta_element = MetaElement(
-            self.project,
-            element.name,
-            element_kind,
-            elt_provenance,
-            meta_sources,
-            node.get_mapping(Symbol.CONFIG, default={}),
-            node.get_mapping(Symbol.VARIABLES, default={}),
-            node.get_mapping(Symbol.ENVIRONMENT, default={}),
-            node.get_str_list(Symbol.ENV_NOCACHE, default=[]),
-            node.get_mapping(Symbol.PUBLIC, default={}),
-            node.get_mapping(Symbol.SANDBOX, default={}),
-            element_kind in ("junction", "link"),
-        )
-
-        # Cache it now, make sure it's already there before recursing
-        self._meta_elements[element.name] = meta_element
-        if self.load_context.task and report_progress:
-            self.load_context.task.add_current_progress()
-
-        return meta_element
-
     ###########################################
     #            Private Methods              #
     ###########################################
@@ -553,52 +469,6 @@ class Loader:
                 check_elements.remove(this_element)
                 validated.add(this_element)
 
-    # _collect_element()
-    #
-    # Collect the toplevel elements we have
-    #
-    # Args:
-    #    top_element (LoadElement): The element for which to load a MetaElement
-    #
-    # Returns:
-    #    (MetaElement): A fully loaded MetaElement
-    #
-    def _collect_element(self, top_element):
-        element_queue = [top_element]
-        meta_element_queue = [self.collect_element_no_deps(top_element, report_progress=True)]
-
-        while element_queue:
-            element = element_queue.pop()
-            meta_element = meta_element_queue.pop()
-
-            if element.meta_done:
-                # This can happen if there are multiple top level targets
-                # in which case, we simply skip over this element.
-                continue
-
-            for dep in element.dependencies:
-
-                loader = dep.element._loader
-                name = dep.element.name
-
-                try:
-                    meta_dep = loader._meta_elements[name]
-                except KeyError:
-                    meta_dep = loader.collect_element_no_deps(dep.element, report_progress=True)
-                    element_queue.append(dep.element)
-                    meta_element_queue.append(meta_dep)
-
-                if dep.dep_type != "runtime":
-                    meta_element.build_dependencies.append(meta_dep)
-                if dep.dep_type != "build":
-                    meta_element.dependencies.append(meta_dep)
-                if dep.strict:
-                    meta_element.strict_dependencies.append(meta_dep)
-
-            element.meta_done = True
-
-        return self._meta_elements[top_element.name]
-
     # _search_for_override():
     #
     # Search parent projects for an overridden subproject to replace this junction.
@@ -711,20 +581,9 @@ class Loader:
         if not load_subprojects:
             return None
 
-        # meta junction element
-        #
-        # Note that junction elements are not allowed to have
-        # dependencies, so disabling progress reporting here should
-        # have no adverse effects - the junction element itself cannot
-        # be depended on, so it would be confusing for its load to
-        # show up in logs.
-        #
-        # Any task counting *inside* the junction will be handled by
-        # its loader.
-        meta_element = self.collect_element_no_deps(self._elements[filename])
-        if meta_element.kind != "junction":
+        if load_element.kind != "junction":
             raise LoadError(
-                "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind),
+                "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, load_element.kind),
                 LoadErrorReason.INVALID_DATA,
             )
 
@@ -748,7 +607,7 @@ class Loader:
                 "{}: Dependencies are forbidden for 'junction' elements".format(p), LoadErrorReason.INVALID_JUNCTION
             )
 
-        element = Element._new_from_meta(meta_element)
+        element = Element._new_from_load_element(load_element)
         element._initialize_state()
 
         # Handle the case where a subproject has no ref
diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py
deleted file mode 100644
index 1c1f6feb2..000000000
--- a/src/buildstream/_loader/metaelement.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-#  Copyright (C) 2016 Codethink Limited
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 2 of the License, or (at your option) any later version.
-#
-#  This library is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
-#  Lesser General Public License for more details.
-#
-#  You should have received a copy of the GNU Lesser General Public
-#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-#  Authors:
-#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
-
-from ..node import Node
-
-
-class MetaElement:
-
-    # MetaElement()
-    #
-    # An abstract object holding data suitable for constructing an Element
-    #
-    # Args:
-    #    project: The project that contains the element
-    #    name: The resolved element name
-    #    kind: The element kind
-    #    provenance: The provenance of the element
-    #    sources: An array of MetaSource objects
-    #    config: The configuration data for the element
-    #    variables: The variables declared or overridden on this element
-    #    environment: The environment variables declared or overridden on this element
-    #    env_nocache: List of environment vars which should not be considered in cache keys
-    #    public: Public domain data dictionary
-    #    sandbox: Configuration specific to the sandbox environment
-    #    first_pass: The element is to be loaded with first pass configuration (junction)
-    #
-    def __init__(
-        self,
-        project,
-        name,
-        kind=None,
-        provenance=None,
-        sources=None,
-        config=None,
-        variables=None,
-        environment=None,
-        env_nocache=None,
-        public=None,
-        sandbox=None,
-        first_pass=False,
-    ):
-        self.project = project
-        self.name = name
-        self.kind = kind
-        self.provenance = provenance
-        self.sources = sources
-        self.config = config or Node.from_dict({})
-        self.variables = variables or Node.from_dict({})
-        self.environment = environment or Node.from_dict({})
-        self.env_nocache = env_nocache or []
-        self.public = public or Node.from_dict({})
-        self.sandbox = sandbox or Node.from_dict({})
-        self.build_dependencies = []
-        self.dependencies = []
-        self.strict_dependencies = []
-        self.first_pass = first_pass
-        self.is_junction = kind in ("junction", "link")
diff --git a/src/buildstream/_loader/metasource.py b/src/buildstream/_loader/metasource.py
index ddaa538f5..d49ae3d82 100644
--- a/src/buildstream/_loader/metasource.py
+++ b/src/buildstream/_loader/metasource.py
@@ -32,12 +32,12 @@ class MetaSource:
     #    config: The configuration data for the source
     #    first_pass: This source will be used with first project pass configuration (used for junctions).
     #
-    def __init__(self, element_name, element_index, element_kind, kind, config, directory):
+    def __init__(self, element_name, element_index, element_kind, kind, config, directory, first_pass):
         self.element_name = element_name
         self.element_index = element_index
         self.element_kind = element_kind
         self.kind = kind
         self.config = config
         self.directory = directory
-        self.first_pass = False
+        self.first_pass = first_pass
         self.provenance = config.get_provenance()