diff options
-rw-r--r-- | src/buildstream/_artifactproject.py | 88 | ||||
-rw-r--r-- | src/buildstream/_project.py | 154 | ||||
-rw-r--r-- | src/buildstream/element.py | 6 |
3 files changed, 181 insertions, 67 deletions
diff --git a/src/buildstream/_artifactproject.py b/src/buildstream/_artifactproject.py new file mode 100644 index 000000000..b8153b06d --- /dev/null +++ b/src/buildstream/_artifactproject.py @@ -0,0 +1,88 @@ +# +# Copyright (C) 2020 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# +from contextlib import suppress +from typing import TYPE_CHECKING + +from ._project import Project +from ._context import Context +from ._loader import Loader + +if TYPE_CHECKING: + from typing import Dict + + +# ArtifactProject() +# +# A project instance to be used as the project for an ArtifactElement. +# +# This is basically a simplified Project implementation which ensures that +# we do not accidentally infer any data from a possibly present local project +# when processing an ArtifactElement. +# +# Args: +# project_name: The name of this project +# +class ArtifactProject(Project): + + __loaded_artifact_projects = {} # type: Dict[str, ArtifactProject] + + def __init__(self, project_name: str, context: Context): + + # + # Chain up to the Project constructor, and allow it to initialize + # without loading anything + # + super().__init__(None, context, search_for_project=False) + + # Fill in some necessities + # + self.name = project_name + self.element_path = "" # This needs to be set to avoid Loader crashes + self.loader = Loader(self) + + # get_artifact_project(): + # + # Gets a reference to an ArtifactProject for the given + # project name, possibly instantiating one if needed. + # + # Args: + # project_name: The project name + # context: The Context + # + # Returns: + # An ArtifactProject with the given project_name + # + @classmethod + def get_artifact_project(cls, project_name: str, context: Context) -> "ArtifactProject": + with suppress(KeyError): + return cls.__loaded_artifact_projects[project_name] + + project = cls(project_name, context) + cls.__loaded_artifact_projects[project_name] = project + return project + + # clear_project_cache(): + # + # Clears the cache of loaded projects, this can be called directly + # after completing a full load. + # + @classmethod + def clear_project_cache(cls): + cls.__loaded_artifact_projects = {} diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index fd4a44732..2534e0209 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -18,6 +18,8 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Tiago Gomes <tiago.gomes@codethink.co.uk> +from typing import TYPE_CHECKING, Optional, Dict, Union, List + import os import sys from collections import OrderedDict @@ -44,6 +46,10 @@ from ._message import Message, MessageType from ._includes import Includes from ._workspaces import WORKSPACE_PROJECT_FILE +if TYPE_CHECKING: + from .node import ProvenanceInformation, MappingNode + from ._context import Context + from ._remote import RemoteSpec # Project Configuration file _PROJECT_CONF_FILE = "project.conf" @@ -86,95 +92,115 @@ class ProjectConfig: # # The Project Configuration # +# Args: +# directory: The project directory, or None for dummy ArtifactProjects +# context: The invocation context +# junction: The junction Element causing this project to be loaded +# cli_options: The project options specified on the command line +# default_mirror: The default mirror specified on the command line +# parent_loader: The parent loader +# provenance_node: The YAML provenance causing this project to be loaded +# search_for_project: Whether to search for a project directory, e.g. from workspace metadata or parent directories +# load_project: Whether to attempt to load a project.conf +# class Project: def __init__( self, - directory, - context, + directory: Optional[str], + context: "Context", *, - junction=None, - cli_options=None, - default_mirror=None, - parent_loader=None, - provenance_node=None, - search_for_project=True, + junction: Optional[object] = None, + cli_options: Optional[Dict[str, str]] = None, + default_mirror: Optional[str] = None, + parent_loader: Optional[Loader] = None, + provenance_node: Optional["ProvenanceInformation"] = None, + search_for_project: bool = True, + load_project: bool = True, ): + # + # Public members + # + self.name: Optional[str] = None # The project name + self.directory: Optional[str] = directory # The project directory + self.element_path: Optional[str] = None # The project relative element path - # The project name - self.name = None - - self._context = context # The invocation Context, a private member - - # Create the LoadContext here if we are the toplevel project. - if parent_loader: - self.load_context = parent_loader.load_context - else: - self.load_context = LoadContext(self._context) - - if search_for_project: - self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory) - else: - self.directory = directory - self._invoked_from_workspace_element = None - - self._absolute_directory_path = Path(self.directory).resolve() - - # Absolute path to where elements are loaded from within the project - self.element_path = None + self.load_context: LoadContext # The LoadContext + self.loader: Optional[Loader] = None # The loader associated to this project + self.junction: Optional[object] = junction # The junction Element object, if this is a subproject - # ProjectRefs for the main refs and also for junctions - self.refs = ProjectRefs(self.directory, "project.refs") - self.junction_refs = ProjectRefs(self.directory, "junction.refs") + self.ref_storage: Optional[ProjectRefStorage] = None # Where to store source refs + self.refs: Optional[ProjectRefs] = None + self.junction_refs: Optional[ProjectRefs] = None - self.config = ProjectConfig() - self.first_pass_config = ProjectConfig() + self.config: ProjectConfig = ProjectConfig() + self.first_pass_config: ProjectConfig = ProjectConfig() - self.junction = junction # The junction Element object, if this is a subproject + self.base_environment: Union["MappingNode", Dict[str, str]] = {} # The base set of environment variables + self.base_env_nocache: List[str] = [] # The base nocache mask (list) for the environment - self.ref_storage = None # ProjectRefStorage setting - self.base_environment = {} # The base set of environment variables - self.base_env_nocache = None # The base nocache mask (list) for the environment + # Remote specs for communicating with remote services + self.artifact_cache_specs: List["RemoteSpec"] = [] # Artifact caches + self.source_cache_specs: List["RemoteSpec"] = [] # Source caches + self.remote_execution_specs: List["RemoteSpec"] = [] # Remote execution services - self.artifact_cache_specs = None - self.source_cache_specs = None - self.remote_execution_specs = None + self.element_factory: Optional[ElementFactory] = None # ElementFactory for loading elements + self.source_factory: Optional[SourceFactory] = None # SourceFactory for loading sources - self.element_factory = None # ElementFactory for loading elements - self.source_factory = None # SourceFactory for loading sources + self.sandbox: Optional["MappingNode"] = None + self.splits: Optional["MappingNode"] = None # - # Private Members + # Private members # - self._default_targets = None # Default target elements - self._default_mirror = default_mirror # The name of the preferred mirror. + self._context: "Context" = context # The invocation Context + self._invoked_from_workspace_element: Optional[str] = None + self._absolute_directory_path: Optional[Path] = None - self._cli_options = cli_options + self._default_targets: Optional[List[str]] = None # Default target elements + self._default_mirror: Optional[str] = default_mirror # The name of the preferred mirror. + self._cli_options: Optional[Dict[str, str]] = cli_options - self._fatal_warnings = [] # A list of warnings which should trigger an error - - self._shell_command = [] # The default interactive shell command - self._shell_environment = {} # Statically set environment vars - self._shell_host_files = [] # A list of HostMount objects - self._sandbox = None - self._splits = None + self._fatal_warnings: List[str] = [] # A list of warnings which should trigger an error + self._shell_command: List[str] = [] # The default interactive shell command + self._shell_environment: Dict[str, str] = {} # Statically set environment vars + self._shell_host_files: List[str] = [] # A list of HostMount objects # This is a lookup table of lists indexed by project, # the child dictionaries are lists of ScalarNodes indicating # junction names - self._junction_duplicates = {} + self._junction_duplicates: Dict[str, List[str]] = {} # A list of project relative junctions to consider as 'internal', # stored as ScalarNodes. - self._junction_internal = [] + self._junction_internal: List[str] = [] - self._context.add_project(self) + self._partially_loaded: bool = False + self._fully_loaded: bool = False + self._project_includes: Optional[Includes] = None - self._partially_loaded = False - self._fully_loaded = False - self._project_includes = None + # + # Initialization body + # + if parent_loader: + self.load_context = parent_loader.load_context + else: + self.load_context = LoadContext(self._context) - with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, "-")): - self._load(parent_loader=parent_loader, provenance_node=provenance_node) + if search_for_project: + self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory) + + if self.directory: + self._absolute_directory_path = Path(self.directory).resolve() + self.refs = ProjectRefs(self.directory, "project.refs") + self.junction_refs = ProjectRefs(self.directory, "junction.refs") + + self._context.add_project(self) + + if self.directory and load_project: + with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, "-")): + self._load(parent_loader=parent_loader, provenance_node=provenance_node) + else: + self._fully_loaded = True self._partially_loaded = True @@ -873,10 +899,10 @@ class Project: self.base_env_nocache = config.get_str_list("environment-nocache") # Load sandbox configuration - self._sandbox = config.get_mapping("sandbox") + self.sandbox = config.get_mapping("sandbox") # Load project split rules - self._splits = config.get_mapping("split-rules") + self.splits = config.get_mapping("split-rules") # Support backwards compatibility for fail-on-overlap fail_on_overlap = config.get_scalar("fail-on-overlap", None) diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 3ac201d66..c453e3c88 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -2832,9 +2832,9 @@ class Element(Plugin): if first_pass: splits = element_splits.clone() else: - assert project._splits is not None + assert project.splits is not None - splits = project._splits.clone() + splits = project.splits.clone() # Extend project wide split rules with any split rules defined by the element element_splits._composite(splits) @@ -2967,7 +2967,7 @@ class Element(Plugin): if load_element.first_pass: sandbox_config = Node.from_dict({}) else: - sandbox_config = project._sandbox.clone() + sandbox_config = project.sandbox.clone() # The default config is already composited with the project overrides sandbox_defaults = cls.__defaults.get_mapping(Symbol.SANDBOX, default={}) |