diff options
| -rw-r--r-- | src/buildstream/_artifactelement.py | 8 | ||||
| -rw-r--r-- | src/buildstream/_loader/__init__.py | 3 | ||||
| -rw-r--r-- | src/buildstream/_loader/loadelement.pyx | 45 | ||||
| -rw-r--r-- | src/buildstream/_loader/loader.py | 157 | ||||
| -rw-r--r-- | src/buildstream/_loader/metaelement.py | 73 | ||||
| -rw-r--r-- | src/buildstream/_loader/metasource.py | 4 | ||||
| -rw-r--r-- | src/buildstream/_pluginfactory/elementfactory.py | 8 | ||||
| -rw-r--r-- | src/buildstream/_project.py | 10 | ||||
| -rw-r--r-- | src/buildstream/element.py | 216 | ||||
| -rw-r--r-- | src/buildstream/source.py | 3 | ||||
| -rw-r--r-- | tests/internals/loader.py | 4 |
11 files changed, 206 insertions, 325 deletions
diff --git a/src/buildstream/_artifactelement.py b/src/buildstream/_artifactelement.py index 4066cef06..44e52ea90 100644 --- a/src/buildstream/_artifactelement.py +++ b/src/buildstream/_artifactelement.py @@ -22,7 +22,8 @@ from typing import TYPE_CHECKING from . import Element from . import _cachekey from ._exceptions import ArtifactElementError -from ._loader.metaelement import MetaElement +from ._loader import LoadElement +from .node import Node from .types import Scope if TYPE_CHECKING: @@ -49,10 +50,9 @@ class ArtifactElement(Element): self._key = key project = context.get_toplevel_project() - meta = MetaElement(project, element) # NOTE element has no .bst suffix - plugin_conf = None + load_element = LoadElement(Node.from_dict({}), element, project.loader) # NOTE element has no .bst suffix - super().__init__(context, project, meta, plugin_conf) + super().__init__(context, project, load_element, None) # _new_from_artifact_ref(): # diff --git a/src/buildstream/_loader/__init__.py b/src/buildstream/_loader/__init__.py index fd5cac2ae..a4be9cfe5 100644 --- a/src/buildstream/_loader/__init__.py +++ b/src/buildstream/_loader/__init__.py @@ -17,7 +17,8 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +from .types import Symbol from .metasource import MetaSource -from .metaelement import MetaElement +from .loadelement import LoadElement, Dependency from .loadcontext import LoadContext from .loader import Loader diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx index 6cd5b46b7..01334d124 100644 --- a/src/buildstream/_loader/loadelement.pyx +++ b/src/buildstream/_loader/loadelement.pyx @@ -184,9 +184,10 @@ cdef class LoadElement: cdef readonly MappingNode node cdef readonly str name - cdef readonly full_name - cdef public bint meta_done + cdef readonly str full_name + cdef readonly str kind cdef int node_id + cdef readonly bint first_pass cdef readonly object _loader cdef readonly str link_target cdef readonly ProvenanceInformation link_target_provenance @@ -199,10 +200,10 @@ cdef class LoadElement: # # Public members # + self.kind = None # The Element kind self.node = node # The YAML node self.name = filename # The element name self.full_name = None # The element full name (with associated junction) - self.meta_done = False # If the MetaElement for this LoadElement is done self.node_id = _next_synthetic_counter() self.link_target = None # The target of a link element self.link_target_provenance = None # The provenance of the link target @@ -233,13 +234,15 @@ cdef class LoadElement: 'build-depends', 'runtime-depends', ]) + self.kind = node.get_str(Symbol.KIND, default=None) + self.first_pass = self.kind in ("junction", "link") + # # If this is a link, resolve it right away and just # store the link target and provenance # - if self.node.get_str(Symbol.KIND, default=None) == 'link': - meta_element = self._loader.collect_element_no_deps(self) - element = Element._new_from_meta(meta_element) + if self.kind == 'link': + element = Element._new_from_load_element(self) element._initialize_state() # Custom error for link dependencies, since we don't completely @@ -254,6 +257,36 @@ cdef class LoadElement: self.link_target = element.target self.link_target_provenance = element.target_provenance + # We don't count progress for junction elements or link + # as they do not represent real elements in the build graph. + # + # We check for a `None` kind, to avoid reporting progress for + # the virtual toplevel element used to load the pipeline. + # + if self._loader.load_context.task and self.kind is not None and not self.first_pass: + self._loader.load_context.task.add_current_progress() + + # provenance + # + # A property reporting the ProvenanceInformation of the element + # + @property + def provenance(self): + return self.node.get_provenance() + + # project + # + # A property reporting the Project in which this element resides. + # + @property + def project(self): + return self._loader.project + + # junction + # + # A property reporting the junction element accessing this + # element, if any. + # @property def junction(self): return self._loader.project.junction diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index b0f4a4a07..94ee9078b 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -32,8 +32,6 @@ from ._loader import valid_chars_name from .types import Symbol from . import loadelement from .loadelement import LoadElement, Dependency, extract_depends_from_node -from .metaelement import MetaElement -from .metasource import MetaSource from ..types import CoreWarnings, _KeyStrength from .._message import Message, MessageType @@ -41,8 +39,8 @@ from .._message import Message, MessageType # Loader(): # # The Loader class does the heavy lifting of parsing target -# bst files and ultimately transforming them into a list of MetaElements -# with their own MetaSources, ready for instantiation by the core. +# bst files and ultimately transforming them into a list of LoadElements +# ready for instantiation by the core. # # Args: # project (Project): The toplevel Project object @@ -112,7 +110,8 @@ class Loader: # # Raises: LoadError # - # Returns: The toplevel LoadElement + # Returns: + # (list): The corresponding LoadElement instances matching the `targets` # def load(self, targets): @@ -154,7 +153,6 @@ class Loader: with PROFILER.profile(Topics.CIRCULAR_CHECK, "_".join(targets)): self._check_circular_deps(dummy_target) - ret = [] # # Sort direct dependencies of elements by their dependency ordering # @@ -167,18 +165,13 @@ class Loader: with PROFILER.profile(Topics.SORT_DEPENDENCIES, element.name): loadelement.sort_dependencies(element, visited_elements) - # Finally, wrap what we have into LoadElements and return the target - # - ret.append(loader._collect_element(element)) - self._clean_caches() # Cache how many Elements have just been loaded if self.load_context.task: - # Workaround for task potentially being None (because no State object) self.loaded = self.load_context.task.current_progress - return ret + return target_elements # get_loader(): # @@ -252,83 +245,6 @@ class Loader: for parent in self._alternative_parents: yield from foreach_parent(parent) - # collect_element_no_deps() - # - # Collect a single element, without its dependencies, into a meta_element - # - # NOTE: This is declared public in order to share with the LoadElement - # and should not be used outside of that `_loader` module. - # - # Args: - # element (LoadElement): The element for which to load a MetaElement - # report_progress (bool): Whether to report progress for this element, this is - # because we ignore junctions and links when counting - # how many elements we load. - # - # Returns: - # (MetaElement): A partially loaded MetaElement - # - def collect_element_no_deps(self, element, *, report_progress=False): - # Return the already built one, if we already built it - meta_element = self._meta_elements.get(element.name) - if meta_element: - return meta_element - - node = element.node - elt_provenance = node.get_provenance() - meta_sources = [] - - element_kind = node.get_str(Symbol.KIND) - - # if there's a workspace for this element then just append a dummy workspace - # metasource. - workspace = self.load_context.context.get_workspaces().get_workspace(element.full_name) - skip_workspace = True - if workspace: - workspace_node = {"kind": "workspace"} - workspace_node["path"] = workspace.get_absolute_path() - workspace_node["last_build"] = str(workspace.to_dict().get("last_build", "")) - node[Symbol.SOURCES] = [workspace_node] - skip_workspace = False - - sources = node.get_sequence(Symbol.SOURCES, default=[]) - for index, source in enumerate(sources): - kind = source.get_str(Symbol.KIND) - # the workspace source plugin cannot be used unless the element is workspaced - if kind == "workspace" and skip_workspace: - continue - - del source[Symbol.KIND] - - # Directory is optional - directory = source.get_str(Symbol.DIRECTORY, default=None) - if directory: - del source[Symbol.DIRECTORY] - meta_source = MetaSource(element.name, index, element_kind, kind, source, directory) - meta_sources.append(meta_source) - - meta_element = MetaElement( - self.project, - element.name, - element_kind, - elt_provenance, - meta_sources, - node.get_mapping(Symbol.CONFIG, default={}), - node.get_mapping(Symbol.VARIABLES, default={}), - node.get_mapping(Symbol.ENVIRONMENT, default={}), - node.get_str_list(Symbol.ENV_NOCACHE, default=[]), - node.get_mapping(Symbol.PUBLIC, default={}), - node.get_mapping(Symbol.SANDBOX, default={}), - element_kind in ("junction", "link"), - ) - - # Cache it now, make sure it's already there before recursing - self._meta_elements[element.name] = meta_element - if self.load_context.task and report_progress: - self.load_context.task.add_current_progress() - - return meta_element - ########################################### # Private Methods # ########################################### @@ -553,52 +469,6 @@ class Loader: check_elements.remove(this_element) validated.add(this_element) - # _collect_element() - # - # Collect the toplevel elements we have - # - # Args: - # top_element (LoadElement): The element for which to load a MetaElement - # - # Returns: - # (MetaElement): A fully loaded MetaElement - # - def _collect_element(self, top_element): - element_queue = [top_element] - meta_element_queue = [self.collect_element_no_deps(top_element, report_progress=True)] - - while element_queue: - element = element_queue.pop() - meta_element = meta_element_queue.pop() - - if element.meta_done: - # This can happen if there are multiple top level targets - # in which case, we simply skip over this element. - continue - - for dep in element.dependencies: - - loader = dep.element._loader - name = dep.element.name - - try: - meta_dep = loader._meta_elements[name] - except KeyError: - meta_dep = loader.collect_element_no_deps(dep.element, report_progress=True) - element_queue.append(dep.element) - meta_element_queue.append(meta_dep) - - if dep.dep_type != "runtime": - meta_element.build_dependencies.append(meta_dep) - if dep.dep_type != "build": - meta_element.dependencies.append(meta_dep) - if dep.strict: - meta_element.strict_dependencies.append(meta_dep) - - element.meta_done = True - - return self._meta_elements[top_element.name] - # _search_for_override(): # # Search parent projects for an overridden subproject to replace this junction. @@ -711,20 +581,9 @@ class Loader: if not load_subprojects: return None - # meta junction element - # - # Note that junction elements are not allowed to have - # dependencies, so disabling progress reporting here should - # have no adverse effects - the junction element itself cannot - # be depended on, so it would be confusing for its load to - # show up in logs. - # - # Any task counting *inside* the junction will be handled by - # its loader. - meta_element = self.collect_element_no_deps(self._elements[filename]) - if meta_element.kind != "junction": + if load_element.kind != "junction": raise LoadError( - "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind), + "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, load_element.kind), LoadErrorReason.INVALID_DATA, ) @@ -748,7 +607,7 @@ class Loader: "{}: Dependencies are forbidden for 'junction' elements".format(p), LoadErrorReason.INVALID_JUNCTION ) - element = Element._new_from_meta(meta_element) + element = Element._new_from_load_element(load_element) element._initialize_state() # Handle the case where a subproject has no ref diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py deleted file mode 100644 index 1c1f6feb2..000000000 --- a/src/buildstream/_loader/metaelement.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (C) 2016 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# -# Authors: -# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> - -from ..node import Node - - -class MetaElement: - - # MetaElement() - # - # An abstract object holding data suitable for constructing an Element - # - # Args: - # project: The project that contains the element - # name: The resolved element name - # kind: The element kind - # provenance: The provenance of the element - # sources: An array of MetaSource objects - # config: The configuration data for the element - # variables: The variables declared or overridden on this element - # environment: The environment variables declared or overridden on this element - # env_nocache: List of environment vars which should not be considered in cache keys - # public: Public domain data dictionary - # sandbox: Configuration specific to the sandbox environment - # first_pass: The element is to be loaded with first pass configuration (junction) - # - def __init__( - self, - project, - name, - kind=None, - provenance=None, - sources=None, - config=None, - variables=None, - environment=None, - env_nocache=None, - public=None, - sandbox=None, - first_pass=False, - ): - self.project = project - self.name = name - self.kind = kind - self.provenance = provenance - self.sources = sources - self.config = config or Node.from_dict({}) - self.variables = variables or Node.from_dict({}) - self.environment = environment or Node.from_dict({}) - self.env_nocache = env_nocache or [] - self.public = public or Node.from_dict({}) - self.sandbox = sandbox or Node.from_dict({}) - self.build_dependencies = [] - self.dependencies = [] - self.strict_dependencies = [] - self.first_pass = first_pass - self.is_junction = kind in ("junction", "link") diff --git a/src/buildstream/_loader/metasource.py b/src/buildstream/_loader/metasource.py index ddaa538f5..d49ae3d82 100644 --- a/src/buildstream/_loader/metasource.py +++ b/src/buildstream/_loader/metasource.py @@ -32,12 +32,12 @@ class MetaSource: # config: The configuration data for the source # first_pass: This source will be used with first project pass configuration (used for junctions). # - def __init__(self, element_name, element_index, element_kind, kind, config, directory): + def __init__(self, element_name, element_index, element_kind, kind, config, directory, first_pass): self.element_name = element_name self.element_index = element_index self.element_kind = element_kind self.kind = kind self.config = config self.directory = directory - self.first_pass = False + self.first_pass = first_pass self.provenance = config.get_provenance() diff --git a/src/buildstream/_pluginfactory/elementfactory.py b/src/buildstream/_pluginfactory/elementfactory.py index 9854c1a5c..7d94c2541 100644 --- a/src/buildstream/_pluginfactory/elementfactory.py +++ b/src/buildstream/_pluginfactory/elementfactory.py @@ -39,7 +39,7 @@ class ElementFactory(PluginFactory): # Args: # context (object): The Context object for processing # project (object): The project object - # meta (object): The loaded MetaElement + # load_element (object): The LoadElement # # Returns: A newly created Element object of the appropriate kind # @@ -47,7 +47,7 @@ class ElementFactory(PluginFactory): # PluginError (if the kind lookup failed) # LoadError (if the element itself took issue with the config) # - def create(self, context, project, meta): - element_type, default_config = self.lookup(context.messenger, meta.kind, meta.provenance) - element = element_type(context, project, meta, default_config) + def create(self, context, project, load_element): + element_type, default_config = self.lookup(context.messenger, load_element.kind, load_element.provenance) + element = element_type(context, project, load_element, default_config) return element diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 3562ea5d4..87772995a 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -345,13 +345,13 @@ class Project: # Instantiate and return an element # # Args: - # meta (MetaElement): The loaded MetaElement + # load_element (LoadElement): The LoadElement # # Returns: # (Element): A newly created Element object of the appropriate kind # - def create_element(self, meta): - return self.element_factory.create(self._context, self, meta) + def create_element(self, load_element): + return self.element_factory.create(self._context, self, load_element) # create_source() # @@ -426,13 +426,13 @@ class Project: with self._context.messenger.simple_task("Loading elements", silent_nested=True) as task: self.load_context.set_task(task) - meta_elements = self.loader.load(targets) + load_elements = self.loader.load(targets) self.load_context.set_task(None) with self._context.messenger.simple_task("Resolving elements") as task: if task: task.set_maximum_progress(self.loader.loaded) - elements = [Element._new_from_meta(meta, task) for meta in meta_elements] + elements = [Element._new_from_load_element(load_element, task) for load_element in load_elements] Element._clear_meta_elements_cache() diff --git a/src/buildstream/element.py b/src/buildstream/element.py index f1e909bce..f7d49d57f 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -80,7 +80,7 @@ import copy import warnings from collections import OrderedDict import contextlib -from contextlib import contextmanager +from contextlib import contextmanager, suppress from functools import partial from itertools import chain import string @@ -106,6 +106,7 @@ from .sandbox._sandboxremote import SandboxRemote from .types import CoreWarnings, Scope, _CacheBuildTrees, _KeyStrength from ._artifact import Artifact from ._elementsources import ElementSources +from ._loader import Symbol, MetaSource from .storage.directory import Directory from .storage._filebaseddirectory import FileBasedDirectory @@ -120,7 +121,7 @@ if TYPE_CHECKING: from .sandbox import Sandbox from .source import Source from ._context import Context - from ._loader.metaelement import MetaElement + from ._loader import LoadElement from ._project import Project # pylint: enable=cyclic-import @@ -157,8 +158,8 @@ class Element(Plugin): # The defaults from the yaml file and project __defaults = None - # A hash of Element by MetaElement - __instantiated_elements = {} # type: Dict[MetaElement, Element] + # A hash of Element by LoadElement + __instantiated_elements = {} # type: Dict[LoadElement, Element] # A list of (source, ref) tuples which were redundantly specified __redundant_source_refs = [] # type: List[Tuple[Source, SourceRef]] @@ -197,15 +198,17 @@ class Element(Plugin): """Whether the element produces an artifact when built. """ - def __init__(self, context: "Context", project: "Project", meta: "MetaElement", plugin_conf: Dict[str, Any]): + def __init__( + self, context: "Context", project: "Project", load_element: "LoadElement", plugin_conf: Dict[str, Any] + ): self.__cache_key_dict = None # Dict for cache key calculation self.__cache_key = None # Our cached cache key - super().__init__(meta.name, context, project, meta.provenance, "element") + super().__init__(load_element.name, context, project, load_element.provenance, "element") # Ensure the project is fully loaded here rather than later on - if not meta.is_junction: + if not load_element.first_pass: project.ensure_fully_loaded() self.project_name = self._get_project().name @@ -270,44 +273,44 @@ class Element(Plugin): self.__resolved_initial_state = False # Whether the initial state of the Element has been resolved # Ensure we have loaded this class's defaults - self.__init_defaults(project, plugin_conf, meta.kind, meta.is_junction) + self.__init_defaults(project, plugin_conf, load_element.kind, load_element.first_pass) # Collect the composited variables and resolve them - variables = self.__extract_variables(project, meta) + variables = self.__extract_variables(project, load_element) variables["element-name"] = self.name self.__variables = Variables(variables) - if not meta.is_junction: + if not load_element.first_pass: self.__variables.check() # Collect the composited environment now that we have variables - unexpanded_env = self.__extract_environment(project, meta) + unexpanded_env = self.__extract_environment(project, load_element) self.__variables.expand(unexpanded_env) self.__environment = unexpanded_env.strip_node_info() # Collect the environment nocache blacklist list - nocache = self.__extract_env_nocache(project, meta) + nocache = self.__extract_env_nocache(project, load_element) self.__env_nocache = nocache # Grab public domain data declared for this instance - self.__public = self.__extract_public(meta) + self.__public = self.__extract_public(load_element) self.__variables.expand(self.__public) self.__dynamic_public = None # Collect the composited element configuration and # ask the element to configure itself. - self.__config = self.__extract_config(meta) + self.__config = self.__extract_config(load_element) self.__variables.expand(self.__config) self._configure(self.__config) # Extract remote execution URL - if meta.is_junction: + if load_element.first_pass: self.__remote_execution_specs = None else: self.__remote_execution_specs = project.remote_execution_specs # Extract Sandbox config - sandbox_config = self.__extract_sandbox_config(project, meta) + sandbox_config = self.__extract_sandbox_config(project, load_element) self.__variables.expand(sandbox_config) self.__sandbox_config = SandboxConfig(sandbox_config, context.platform) @@ -874,61 +877,56 @@ class Element(Plugin): # Private Methods used in BuildStream # ############################################################# - # _new_from_meta(): + # _new_from_load_element(): # # Recursively instantiate a new Element instance, its sources - # and its dependencies from a meta element. + # and its dependencies from a LoadElement. + # + # FIXME: Need to use an iterative algorithm here since recursion + # will limit project dependency depth. # # Args: - # meta (MetaElement): The meta element + # load_element (LoadElement): The LoadElement # task (Task): A task object to report progress to # # Returns: # (Element): A newly created Element instance # @classmethod - def _new_from_meta(cls, meta, task=None): + def _new_from_load_element(cls, load_element, task=None): - if not meta.first_pass: - meta.project.ensure_fully_loaded() + if not load_element.first_pass: + load_element.project.ensure_fully_loaded() - if meta in cls.__instantiated_elements: - return cls.__instantiated_elements[meta] + with suppress(KeyError): + return cls.__instantiated_elements[load_element] - element = meta.project.create_element(meta) - cls.__instantiated_elements[meta] = element + element = load_element.project.create_element(load_element) + cls.__instantiated_elements[load_element] = element - # Instantiate sources and generate their keys - for meta_source in meta.sources: - meta_source.first_pass = meta.is_junction - source = meta.project.create_source(meta_source, variables=element.__variables) + # Load the sources from the LoadElement + element.__load_sources(load_element) - redundant_ref = source._load_ref() + # Instantiate dependencies + for dep in load_element.dependencies: + dependency = Element._new_from_load_element(dep.element, task) - element.__sources.add_source(source) + if dep.dep_type != "runtime": + element.__build_dependencies.append(dependency) + dependency.__reverse_build_deps.add(element) - # Collect redundant refs which occurred at load time - if redundant_ref is not None: - cls.__redundant_source_refs.append((source, redundant_ref)) + if dep.dep_type != "build": + element.__runtime_dependencies.append(dependency) + dependency.__reverse_runtime_deps.add(element) + + if dep.strict: + element.__strict_dependencies.append(dependency) - # Instantiate dependencies - for meta_dep in meta.dependencies: - dependency = Element._new_from_meta(meta_dep, task) - element.__runtime_dependencies.append(dependency) - dependency.__reverse_runtime_deps.add(element) no_of_runtime_deps = len(element.__runtime_dependencies) element.__runtime_deps_without_strict_cache_key = no_of_runtime_deps element.__runtime_deps_without_cache_key = no_of_runtime_deps element.__runtime_deps_uncached = no_of_runtime_deps - for meta_dep in meta.build_dependencies: - dependency = Element._new_from_meta(meta_dep, task) - element.__build_dependencies.append(dependency) - dependency.__reverse_build_deps.add(element) - - if meta_dep in meta.strict_dependencies: - element.__strict_dependencies.append(dependency) - no_of_build_deps = len(element.__build_dependencies) element.__build_deps_without_strict_cache_key = no_of_build_deps element.__build_deps_without_cache_key = no_of_build_deps @@ -2203,6 +2201,63 @@ class Element(Plugin): # Private Local Methods # ############################################################# + # __load_sources() + # + # Load the Source objects from the LoadElement + # + def __load_sources(self, load_element): + project = self._get_project() + workspace = self._get_workspace() + meta_sources = [] + + # If there's a workspace for this element then we just load a workspace + # source plugin instead of the real plugins + if workspace: + workspace_node = {"kind": "workspace"} + workspace_node["path"] = workspace.get_absolute_path() + workspace_node["last_build"] = str(workspace.to_dict().get("last_build", "")) + meta = MetaSource( + self.name, + 0, + self.get_kind(), + "workspace", + Node.from_dict(workspace_node), + None, + load_element.first_pass, + ) + meta_sources.append(meta) + else: + sources = load_element.node.get_sequence(Symbol.SOURCES, default=[]) + for index, source in enumerate(sources): + kind = source.get_scalar(Symbol.KIND) + + # The workspace source plugin is only valid for internal use + if kind.as_str() == "workspace": + raise LoadError( + "{}: Invalid usage of workspace source kind".format(kind.get_provenance()), + LoadErrorReason.INVALID_DATA, + ) + del source[Symbol.KIND] + + # Directory is optional + directory = source.get_str(Symbol.DIRECTORY, default=None) + if directory: + del source[Symbol.DIRECTORY] + meta_source = MetaSource( + self.name, index, self.get_kind(), kind.as_str(), source, directory, load_element.first_pass + ) + meta_sources.append(meta_source) + + for meta_source in meta_sources: + source = project.create_source(meta_source, variables=self.__variables) + redundant_ref = source._load_ref() + + self.__sources.add_source(source) + + # Collect redundant refs which occurred at load time + if redundant_ref is not None: + self.__redundant_source_refs.append((source, redundant_ref)) + # __get_dependency_refs() # # Retrieve the artifact refs of the element's dependencies @@ -2433,13 +2488,13 @@ class Element(Plugin): yield sandbox @classmethod - def __compose_default_splits(cls, project, defaults, is_junction): + def __compose_default_splits(cls, project, defaults, first_pass): - element_public = defaults.get_mapping("public", default={}) + element_public = defaults.get_mapping(Symbol.PUBLIC, default={}) element_bst = element_public.get_mapping("bst", default={}) element_splits = element_bst.get_mapping("split-rules", default={}) - if is_junction: + if first_pass: splits = element_splits.clone() else: assert project._splits is not None @@ -2450,10 +2505,10 @@ class Element(Plugin): element_bst["split-rules"] = splits element_public["bst"] = element_bst - defaults["public"] = element_public + defaults[Symbol.PUBLIC] = element_public @classmethod - def __init_defaults(cls, project, plugin_conf, kind, is_junction): + def __init_defaults(cls, project, plugin_conf, kind, first_pass): # Defaults are loaded once per class and then reused # if cls.__defaults is None: @@ -2468,11 +2523,11 @@ class Element(Plugin): raise e # Special case; compose any element-wide split-rules declarations - cls.__compose_default_splits(project, defaults, is_junction) + cls.__compose_default_splits(project, defaults, first_pass) # Override the element's defaults with element specific # overrides from the project.conf - if is_junction: + if first_pass: elements = project.first_pass_config.element_overrides else: elements = project.element_overrides @@ -2488,29 +2543,30 @@ class Element(Plugin): # creating sandboxes for this element # @classmethod - def __extract_environment(cls, project, meta): - default_env = cls.__defaults.get_mapping("environment", default={}) + def __extract_environment(cls, project, load_element): + default_env = cls.__defaults.get_mapping(Symbol.ENVIRONMENT, default={}) + element_env = load_element.node.get_mapping(Symbol.ENVIRONMENT, default={}) or Node.from_dict({}) - if meta.is_junction: + if load_element.first_pass: environment = Node.from_dict({}) else: environment = project.base_environment.clone() default_env._composite(environment) - meta.environment._composite(environment) + element_env._composite(environment) environment._assert_fully_composited() return environment @classmethod - def __extract_env_nocache(cls, project, meta): - if meta.is_junction: + def __extract_env_nocache(cls, project, load_element): + if load_element.first_pass: project_nocache = [] else: project_nocache = project.base_env_nocache - default_nocache = cls.__defaults.get_str_list("environment-nocache", default=[]) - element_nocache = meta.env_nocache + default_nocache = cls.__defaults.get_str_list(Symbol.ENV_NOCACHE, default=[]) + element_nocache = load_element.node.get_str_list(Symbol.ENV_NOCACHE, default=[]) # Accumulate values from the element default, the project and the element # itself to form a complete list of nocache env vars. @@ -2523,16 +2579,17 @@ class Element(Plugin): # substituting command strings to be run in the sandbox # @classmethod - def __extract_variables(cls, project, meta): - default_vars = cls.__defaults.get_mapping("variables", default={}) + def __extract_variables(cls, project, load_element): + default_vars = cls.__defaults.get_mapping(Symbol.VARIABLES, default={}) + element_vars = load_element.node.get_mapping(Symbol.VARIABLES, default={}) or Node.from_dict({}) - if meta.is_junction: + if load_element.first_pass: variables = project.first_pass_config.base_variables.clone() else: variables = project.base_variables.clone() default_vars._composite(variables) - meta.variables._composite(variables) + element_vars._composite(variables) variables._assert_fully_composited() for var in ("project-name", "element-name", "max-jobs"): @@ -2554,13 +2611,14 @@ class Element(Plugin): # off to element.configure() # @classmethod - def __extract_config(cls, meta): + def __extract_config(cls, load_element): + element_config = load_element.node.get_mapping(Symbol.CONFIG, default={}) or Node.from_dict({}) # The default config is already composited with the project overrides - config = cls.__defaults.get_mapping("config", default={}) + config = cls.__defaults.get_mapping(Symbol.CONFIG, default={}) config = config.clone() - meta.config._composite(config) + element_config._composite(config) config._assert_fully_composited() return config @@ -2568,18 +2626,20 @@ class Element(Plugin): # Sandbox-specific configuration data, to be passed to the sandbox's constructor. # @classmethod - def __extract_sandbox_config(cls, project, meta): - if meta.is_junction: + def __extract_sandbox_config(cls, project, load_element): + element_sandbox = load_element.node.get_mapping(Symbol.SANDBOX, default={}) or Node.from_dict({}) + + if load_element.first_pass: sandbox_config = Node.from_dict({}) else: sandbox_config = project._sandbox.clone() # The default config is already composited with the project overrides - sandbox_defaults = cls.__defaults.get_mapping("sandbox", default={}) + sandbox_defaults = cls.__defaults.get_mapping(Symbol.SANDBOX, default={}) sandbox_defaults = sandbox_defaults.clone() sandbox_defaults._composite(sandbox_config) - meta.sandbox._composite(sandbox_config) + element_sandbox._composite(sandbox_config) sandbox_config._assert_fully_composited() return sandbox_config @@ -2588,14 +2648,16 @@ class Element(Plugin): # elements may extend but whos defaults are defined in the project. # @classmethod - def __extract_public(cls, meta): - base_public = cls.__defaults.get_mapping("public", default={}) + def __extract_public(cls, load_element): + element_public = load_element.node.get_mapping(Symbol.PUBLIC, default={}) or Node.from_dict({}) + + base_public = cls.__defaults.get_mapping(Symbol.PUBLIC, default={}) base_public = base_public.clone() base_bst = base_public.get_mapping("bst", default={}) base_splits = base_bst.get_mapping("split-rules", default={}) - element_public = meta.public.clone() + element_public = element_public.clone() element_bst = element_public.get_mapping("bst", default={}) element_splits = element_bst.get_mapping("split-rules", default={}) diff --git a/src/buildstream/source.py b/src/buildstream/source.py index dc87c35de..ea77a6537 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -1213,10 +1213,9 @@ class Source(Plugin): self.get_kind(), self.__config, self.__directory, + self.__first_pass, ) - meta.first_pass = self.__first_pass - clone = source_kind( context, project, meta, self.__variables, alias_override=(alias, uri), unique_id=self._unique_id ) diff --git a/tests/internals/loader.py b/tests/internals/loader.py index bdce428f0..2da01723b 100644 --- a/tests/internals/loader.py +++ b/tests/internals/loader.py @@ -5,7 +5,7 @@ import pytest from buildstream.exceptions import LoadErrorReason from buildstream._exceptions import LoadError from buildstream._project import Project -from buildstream._loader import MetaElement +from buildstream._loader import LoadElement from tests.testutils import dummy_context @@ -30,7 +30,7 @@ def test_one_file(datafiles): with make_loader(basedir) as loader: element = loader.load(["elements/onefile.bst"])[0] - assert isinstance(element, MetaElement) + assert isinstance(element, LoadElement) assert element.kind == "pony" |