_loader: Adding LoadContexttristan/load-context

Instead of passing around many details though calling signatures
throughout the loader code, create a single LoadContext object
which holds any overall loading state along with any values which
are constant to a full load process.

Overall this patch does:

  * _frontend/app.py: No need to pass Stream.fetch_subprojects() along anymore

  * _loader/loadelement.pyx: collect_element_no_deps() no longer takes a task argument

  * _loader/loader.py: Now the Loader has a `load_context` member, and no more
    `_fetch_subprojects` member or `_context` members

    Further, `rewritable` and `ticker` is no longer passed along through all
    of the recursing calling signatures, and `ticker` itself is finally removed
    because this has been replaced a long time ago with `Task` API from `State`.

  * _pipeline.py: The load() function no longer has a `rewritable` parameter

  * _project.py: The Project() is responsible for creating the toplevel
    LoadContext() if one doesn't exist yet, and this is passed through
    to the Loader() (and also passed to the Project() constructor by the
    Loader() when instantiating subprojects).

  * _stream.py: The `Stream._fetch_subprojects()` is now private and set
    on the project when giving the Project to the Stream in `Stream.set_project()`,
    also the Stream() sets the `rewritable` state on the `LoadContext` at the
    earliest opportunity, as the `Stream()` is the one who decides this detail.

    Further, some double underscore private functions are now regular single
    underscores, there was no reason for this inconsistency.

  * tests/internals/loader.py: Updated for API change

1 files changed, 15 insertions, 9 deletions

diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 3006e2976..e0ddf3d41 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -39,7 +39,7 @@ from .sandbox import SandboxRemote
 from ._pluginfactory import ElementFactory, SourceFactory, load_plugin_origin
 from .types import CoreWarnings
 from ._projectrefs import ProjectRefs, ProjectRefStorage
-from ._loader import Loader
+from ._loader import Loader, LoadContext
 from .element import Element
 from ._message import Message, MessageType
 from ._includes import Includes
@@ -100,7 +100,6 @@ class Project:
         default_mirror=None,
         parent_loader=None,
         search_for_project=True,
-        fetch_subprojects=None
     ):
 
         # The project name
@@ -108,6 +107,12 @@ class Project:
 
         self._context = context  # The invocation Context, a private member
 
+        # Create the LoadContext here if we are the toplevel project.
+        if parent_loader:
+            self.load_context = parent_loader.load_context
+        else:
+            self.load_context = LoadContext(self._context)
+
         if search_for_project:
             self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
         else:
@@ -159,7 +164,7 @@ class Project:
         self._project_includes = None
 
         with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, "-")):
-            self._load(parent_loader=parent_loader, fetch_subprojects=fetch_subprojects)
+            self._load(parent_loader=parent_loader)
 
         self._partially_loaded = True
 
@@ -410,15 +415,16 @@ class Project:
     #
     # Args:
     #    targets (list): Target names
-    #    rewritable (bool): Whether the loaded files should be rewritable
-    #                       this is a bit more expensive due to deep copies
     #
     # Returns:
     #    (list): A list of loaded Element
     #
-    def load_elements(self, targets, *, rewritable=False):
+    def load_elements(self, targets):
+
         with self._context.messenger.simple_task("Loading elements", silent_nested=True) as task:
-            meta_elements = self.loader.load(targets, task, rewritable=rewritable, ticker=None)
+            self.load_context.set_task(task)
+            meta_elements = self.loader.load(targets)
+            self.load_context.set_task(None)
 
         with self._context.messenger.simple_task("Resolving elements") as task:
             if task:
@@ -647,7 +653,7 @@ class Project:
     #
     # Raises: LoadError if there was a problem with the project.conf
     #
-    def _load(self, *, parent_loader=None, fetch_subprojects):
+    def _load(self, *, parent_loader=None):
 
         # Load builtin default
         projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
@@ -694,7 +700,7 @@ class Project:
         # Fatal warnings
         self._fatal_warnings = pre_config_node.get_str_list("fatal-warnings", default=[])
 
-        self.loader = Loader(self._context, self, parent=parent_loader, fetch_subprojects=fetch_subprojects)
+        self.loader = Loader(self, parent=parent_loader)
 
         self._project_includes = Includes(self.loader, copy_tree=False)