From c81f540983decc30055627c19c871f1a307813db Mon Sep 17 00:00:00 2001 From: Valentin David Date: Thu, 17 Jan 2019 13:52:38 +0100 Subject: Add support for sysroot'ed dependencies in BuildElement and ScriptElement --- buildstream/_sysroot_dependency_loader.py | 260 +++++++++++++++++++++ buildstream/buildelement.py | 26 +-- buildstream/element.py | 8 +- buildstream/plugins/elements/script.py | 5 +- buildstream/scriptelement.py | 109 ++------- doc/source/format_declaring.rst | 58 +++++ tests/sysroot_depends/project/elements/a.bst | 4 + tests/sysroot_depends/project/elements/b.bst | 4 + tests/sysroot_depends/project/elements/base.bst | 3 + .../project/elements/base/base-alpine.bst | 12 + .../project/elements/compose-integration.bst | 6 + .../elements/compose-layers-with-sysroot.bst | 13 ++ .../project/elements/compose-layers.bst | 4 + .../project/elements/integration.bst | 13 ++ .../project/elements/layer1-files.bst | 4 + tests/sysroot_depends/project/elements/layer1.bst | 4 + .../project/elements/layer2-files.bst | 4 + tests/sysroot_depends/project/elements/layer2.bst | 22 ++ .../elements/manual-integration-runtime.bst | 14 ++ .../project/elements/manual-integration.bst | 15 ++ .../project/elements/sysroot-integration.bst | 10 + .../project/elements/target-variable.bst | 14 ++ tests/sysroot_depends/project/elements/target.bst | 14 ++ tests/sysroot_depends/project/files/a/a.txt | 1 + tests/sysroot_depends/project/files/b/b.txt | 1 + tests/sysroot_depends/project/files/layer1/1 | 1 + tests/sysroot_depends/project/files/layer2/2 | 1 + tests/sysroot_depends/project/project.conf | 9 + tests/sysroot_depends/sysroot_depends.py | 176 ++++++++++++++ 29 files changed, 707 insertions(+), 108 deletions(-) create mode 100644 buildstream/_sysroot_dependency_loader.py create mode 100644 tests/sysroot_depends/project/elements/a.bst create mode 100644 tests/sysroot_depends/project/elements/b.bst create mode 100644 tests/sysroot_depends/project/elements/base.bst create mode 100644 tests/sysroot_depends/project/elements/base/base-alpine.bst create mode 100644 tests/sysroot_depends/project/elements/compose-integration.bst create mode 100644 tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst create mode 100644 tests/sysroot_depends/project/elements/compose-layers.bst create mode 100644 tests/sysroot_depends/project/elements/integration.bst create mode 100644 tests/sysroot_depends/project/elements/layer1-files.bst create mode 100644 tests/sysroot_depends/project/elements/layer1.bst create mode 100644 tests/sysroot_depends/project/elements/layer2-files.bst create mode 100644 tests/sysroot_depends/project/elements/layer2.bst create mode 100644 tests/sysroot_depends/project/elements/manual-integration-runtime.bst create mode 100644 tests/sysroot_depends/project/elements/manual-integration.bst create mode 100644 tests/sysroot_depends/project/elements/sysroot-integration.bst create mode 100644 tests/sysroot_depends/project/elements/target-variable.bst create mode 100644 tests/sysroot_depends/project/elements/target.bst create mode 100644 tests/sysroot_depends/project/files/a/a.txt create mode 100644 tests/sysroot_depends/project/files/b/b.txt create mode 100644 tests/sysroot_depends/project/files/layer1/1 create mode 100644 tests/sysroot_depends/project/files/layer2/2 create mode 100644 tests/sysroot_depends/project/project.conf create mode 100644 tests/sysroot_depends/sysroot_depends.py diff --git a/buildstream/_sysroot_dependency_loader.py b/buildstream/_sysroot_dependency_loader.py new file mode 100644 index 000000000..c9b348ceb --- /dev/null +++ b/buildstream/_sysroot_dependency_loader.py @@ -0,0 +1,260 @@ +# +# Copyright (C) 2019 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Valentin David + +import os +import itertools +from collections.abc import Mapping + +from .dependency_loader import DependencyLoader, Dependency +from . import Element, ElementError, Scope, SandboxFlags + + +# SysrootDependencyLoader(): +# +# `SysrootDependencyLoader` implements a `DependencyLoader` to extract +# sysroot'ed dependencies. +class SysrootDependencyLoader(DependencyLoader): + + def get_dependencies(self, node): + sysroots = self.node_get_member(node, list, 'sysroots', default=[]) + dependencies = [] + + for sysroot in sysroots: + depends = self.node_get_member(sysroot, list, 'depends', default=[]) + build_depends = self.node_get_member(sysroot, list, 'build-depends', default=[]) + depends_iter = itertools.product(['all'], depends) + build_depends_iter = itertools.product(['build'], build_depends) + for default_type, dep in itertools.chain(depends_iter, build_depends_iter): + if isinstance(dep, Mapping): + provenance = self.node_provenance(dep) + filename = self.node_get_member(node, str, 'filename') + dep_type = self.node_get_member(node, str, 'type', default=default_type) + junction = self.node_get_member(node, str, 'junction', default=None) + dependencies.append(Dependency(filename, dep_type=dep_type, + junction=junction, provenance=provenance)) + else: + provenance = self.node_provenance(sysroot) + dependencies.append(Dependency(dep, dep_type=default_type, provenance=provenance)) + + return dependencies + + +# SysrootHelper(): +# +# `SysrootHelper` should be used in element plugins that use +# `SysrootDependencyLoader` as dependency loader. It provides +# The implementation for staging. +class SysrootHelper: + + CONFIG_KEYS = ['sysroots'] + __layout = [] + + def __init__(self, element, node): + + self.__element = element + + for sysroot in self.__element.node_get_member(node, list, 'sysroots', []): + self.__element.node_validate(sysroot, ['path', 'depends', 'build-depends']) + path = self.__element.node_subst_member(sysroot, 'path') + depends = self.__element.node_get_member(sysroot, list, 'depends', default=[]) + build_depends = self.__element.node_get_member(sysroot, list, 'build-depends', default=[]) + for dep in itertools.chain(depends, build_depends): + if isinstance(dep, Mapping): + self.__element.node_validate(dep, ['filename', 'type', 'junction']) + filename = self.__element.node_get_member(dep, str, 'filename') + junction = self.__element.node_get_member(dep, str, 'junction', default=None) + else: + filename = dep + junction = None + self.layout_add(filename, path, junction=junction) + + # layout_add(): + # + # Adds a destination where a dependency should be staged + # + # Args: + # element (str): Element name of the dependency + # destination (str): Path where element will be staged + # junction (str): Junction of the dependency + # + # If `junction` is None, then the dependency should be in the same + # project as the current element. + # + # If `junction` is ignored or `Element.IGNORE_JUNCTION`, the + # junction of the dependency is not checked. This is for backward + # compliancy and should not be used. + # + # If `element` is None, the destination will just + # be marked in the sandbox. + def layout_add(self, element, destination, *, junction=Element.IGNORE_JUNCTION): + # + # Even if this is an empty list by default, make sure that its + # instance data instead of appending stuff directly onto class data. + # + if not self.__layout: + self.__layout = [] + item = {'element': element, + 'destination': destination} + if junction is not Element.IGNORE_JUNCTION: + item['junction'] = junction + self.__layout.append(item) + + # validate(): + # + # Verify that elements in layouts are dependencies. + # + # Raises: + # (ElementError): When a element is not in the dependencies + # + # This method is only useful when SysrootHelper.layout_add + # has been called directly. + # + # This should be called in implementation of Plugin.preflight. + def validate(self): + if self.__layout: + # Cannot proceed if layout specifies an element that isn't part + # of the dependencies. + for item in self.__layout: + if not item['element']: + if not self.__search(item): + raise ElementError("{}: '{}' in layout not found in dependencies" + .format(self.__element, item['element'])) + + # stage(): + # + # Stage dependencies and integrate root dependencies + # + # Args: + # stage_all (bool): Whether to stage all dependencies, not just the ones mapped + # + def stage(self, sandbox, stage_all): + + staged = set() + sysroots = {} + + for item in self.__layout: + + # Skip layout members which dont stage an element + if not item['element']: + continue + + element = self.__search(item) + staged.add(element) + if item['destination'] not in sysroots: + sysroots[item['destination']] = [element] + else: + sysroots[item['destination']].append(element) + + if stage_all or not self.__layout: + for build_dep in self.__element.dependencies(Scope.BUILD, recurse=False): + if build_dep in staged: + continue + if '/' not in sysroots: + sysroots['/'] = [build_dep] + else: + sysroots['/'].append(build_dep) + + for sysroot, deps in sysroots.items(): + with self.__element.timed_activity("Staging dependencies at {}".format(sysroot), silent_nested=True): + if sysroot != '/': + virtual_dstdir = sandbox.get_virtual_directory() + virtual_dstdir.descend(sysroot.lstrip(os.sep).split(os.sep), create=True) + all_deps = set() + for dep in deps: + for run_dep in dep.dependencies(Scope.RUN): + all_deps.add(run_dep) + self.__element.stage_dependency_artifacts(sandbox, Scope.BUILD, path=sysroot, dependencies=all_deps) + + with sandbox.batch(SandboxFlags.NONE): + for item in self.__layout: + + # Skip layout members which dont stage an element + if not item['element']: + continue + + element = self.__search(item) + + # Integration commands can only be run for elements staged to / + if item['destination'] == '/': + with self.__element.timed_activity("Integrating {}".format(element.name), + silent_nested=True): + for dep in element.dependencies(Scope.RUN): + element.integrate(sandbox) + + if stage_all or not self.__layout: + for build_dep in self.__element.dependencies(Scope.BUILD, recurse=False): + if build_dep in staged: + continue + + with self.__element.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True): + for dep in build_dep.dependencies(Scope.RUN): + dep.integrate(sandbox) + + # has_sysroots(): + # + # Tells whether any element has been mapped + # + # Returns: + # (bool): Whether any element has been mapped + def has_sysroots(self): + return bool(self.__layout) + + # get_unique_key(): + # + # Returns a value usable for an element unique key + # + # Returns: + # (dict): A dictionary that uniquely identify the mapping configuration + def get_unique_key(self): + return self.__layout + + # configure_sandbox(): + # + # Configure the sandbox. Mark required directories in the sandbox. + # + # Args: + # extra_directories (list(str)): Extra directories to mark + # + # Because Sandbox.mark_directory should be called + # only once, marked directories should passed as `extra_directories` + # instead of being marked directly. + def configure_sandbox(self, sandbox, extra_directories): + + directories = {directory: False for directory in extra_directories} + + for item in self.__layout: + destination = item['destination'] + was_artifact = directories.get(destination, False) + directories[destination] = item['element'] or was_artifact + + for directory, artifact in directories.items(): + # Root does not need to be marked as it is always mounted + # with artifact (unless explicitly marked non-artifact) + if directory != '/': + sandbox.mark_directory(directory, artifact=artifact) + + # + # Private methods + # + + def __search(self, item): + if 'junction' in item: + return self.__element.search(Scope.BUILD, item['element'], junction=item['junction']) + else: + return self.__element.search(Scope.BUILD, item['element']) diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py index 6ef060f12..126c966b8 100644 --- a/buildstream/buildelement.py +++ b/buildstream/buildelement.py @@ -135,8 +135,10 @@ artifact collection purposes. """ import os -from . import Element, Scope + +from . import Element from . import SandboxFlags +from ._sysroot_dependency_loader import SysrootDependencyLoader, SysrootHelper # This list is preserved because of an unfortunate situation, we @@ -157,17 +159,20 @@ _command_steps = ['configure-commands', class BuildElement(Element): + DEPENDENCY_LOADER = SysrootDependencyLoader + ############################################################# # Abstract Method Implementations # ############################################################# def configure(self, node): self.__commands = {} # pylint: disable=attribute-defined-outside-init + self.__sysroots = SysrootHelper(self, node) # pylint: disable=attribute-defined-outside-init # FIXME: Currently this forcefully validates configurations # for all BuildElement subclasses so they are unable to # extend the configuration - self.node_validate(node, _command_steps) + self.node_validate(node, _command_steps + SysrootHelper.CONFIG_KEYS) for command_name in _legacy_command_steps: if command_name in _command_steps: @@ -191,6 +196,9 @@ class BuildElement(Element): if self.get_variable('notparallel'): dictionary['notparallel'] = True + if self.__sysroots.has_sysroots(): + dictionary['sysroots'] = self.__sysroots.get_unique_key() + return dictionary def configure_sandbox(self, sandbox): @@ -198,8 +206,8 @@ class BuildElement(Element): install_root = self.get_variable('install-root') # Tell the sandbox to mount the build root and install root - sandbox.mark_directory(build_root) - sandbox.mark_directory(install_root) + self.__sysroots.configure_sandbox(sandbox, [build_root, + install_root]) # Allow running all commands in a specified subdirectory command_subdir = self.get_variable('command-subdir') @@ -217,15 +225,7 @@ class BuildElement(Element): def stage(self, sandbox): - # Stage deps in the sandbox root - with self.timed_activity("Staging dependencies", silent_nested=True): - self.stage_dependency_artifacts(sandbox, Scope.BUILD) - - # Run any integration commands provided by the dependencies - # once they are all staged and ready - with sandbox.batch(SandboxFlags.NONE, label="Integrating sandbox"): - for dep in self.dependencies(Scope.BUILD): - dep.integrate(sandbox) + self.__sysroots.stage(sandbox, True) # Stage sources in the build root self.stage_sources(sandbox, self.get_variable('build-root')) diff --git a/buildstream/element.py b/buildstream/element.py index fcf7c15eb..d28390f0d 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -677,7 +677,8 @@ class Element(Plugin): return link_result.combine(copy_result) def stage_dependency_artifacts(self, sandbox, scope, *, path=None, - include=None, exclude=None, orphans=True): + include=None, exclude=None, orphans=True, + dependencies=None): """Stage element dependencies in scope This is primarily a convenience wrapper around @@ -692,6 +693,7 @@ class Element(Plugin): include (list): An optional list of domains to include files from exclude (list): An optional list of domains to exclude files from orphans (bool): Whether to include files not spoken for by split domains + dependencies (list): An optional list of dependencies to stage Raises: (:class:`.ElementError`): If any of the dependencies in `scope` have not @@ -707,7 +709,9 @@ class Element(Plugin): if self.__can_build_incrementally() and workspace.last_successful: old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful) - for dep in self.dependencies(scope): + if dependencies is None: + dependencies = self.dependencies(scope) + for dep in dependencies: # If we are workspaced, and we therefore perform an # incremental build, we must ensure that we update the mtimes # of any files created by our dependencies since the last diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py index 6c33ecf95..657584baa 100644 --- a/buildstream/plugins/elements/script.py +++ b/buildstream/plugins/elements/script.py @@ -36,6 +36,7 @@ The default configuration and possible options are as such: """ import buildstream +from buildstream._sysroot_dependency_loader import DependencyLoader # Element implementation for the 'script' kind. @@ -46,6 +47,8 @@ class ScriptElement(buildstream.ScriptElement): BST_VIRTUAL_DIRECTORY = True def configure(self, node): + super().configure(node) + for n in self.node_get_member(node, list, 'layout', []): dst = self.node_subst_member(n, 'destination') elm = self.node_subst_member(n, 'element', None) @@ -53,7 +56,7 @@ class ScriptElement(buildstream.ScriptElement): self.node_validate(node, [ 'commands', 'root-read-only', 'layout' - ]) + ] + self.COMMON_CONFIG_KEYS) cmds = self.node_subst_list(node, "commands") self.add_commands("commands", cmds) diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py index 697cd2822..703c5a938 100644 --- a/buildstream/scriptelement.py +++ b/buildstream/scriptelement.py @@ -35,7 +35,8 @@ implementations. import os from collections import OrderedDict -from . import Element, ElementError, Scope, SandboxFlags +from . import Element, SandboxFlags +from ._sysroot_dependency_loader import SysrootDependencyLoader, SysrootHelper class ScriptElement(Element): @@ -43,7 +44,6 @@ class ScriptElement(Element): __cwd = "/" __root_read_only = False __commands = None - __layout = [] # The compose element's output is its dependencies, so # we must rebuild if the dependencies change even when @@ -59,6 +59,15 @@ class ScriptElement(Element): # added, to reduce the potential for confusion BST_FORBID_SOURCES = True + COMMON_CONFIG_KEYS = SysrootHelper.CONFIG_KEYS + + DEPENDENCY_LOADER = SysrootDependencyLoader + + def configure(self, node): + + self.__stage_all = True # pylint: disable=attribute-defined-outside-init + self.__sysroots = SysrootHelper(self, node) # pylint: disable=attribute-defined-outside-init + def set_work_dir(self, work_dir=None): """Sets the working dir @@ -134,14 +143,8 @@ class ScriptElement(Element): In the case that no element is specified, a read-write directory will be made available at the specified location. """ - # - # Even if this is an empty list by default, make sure that its - # instance data instead of appending stuff directly onto class data. - # - if not self.__layout: - self.__layout = [] - self.__layout.append({"element": element, - "destination": destination}) + self.__stage_all = False # pylint: disable=attribute-defined-outside-init + self.__sysroots.layout_add(element, destination) def add_commands(self, group_name, command_list): """Adds a list of commands under the group-name. @@ -164,32 +167,15 @@ class ScriptElement(Element): self.__commands = OrderedDict() self.__commands[group_name] = command_list - def __validate_layout(self): - if self.__layout: - # Cannot proceeed if layout is used, but none are for "/" - root_defined = any([(entry['destination'] == '/') for entry in self.__layout]) - if not root_defined: - raise ElementError("{}: Using layout, but none are staged as '/'" - .format(self)) - - # Cannot proceed if layout specifies an element that isn't part - # of the dependencies. - for item in self.__layout: - if item['element']: - if not self.search(Scope.BUILD, item['element']): - raise ElementError("{}: '{}' in layout not found in dependencies" - .format(self, item['element'])) - def preflight(self): - # The layout, if set, must make sense. - self.__validate_layout() + self.__sysroots.validate() def get_unique_key(self): return { 'commands': self.__commands, 'cwd': self.__cwd, 'install-root': self.__install_root, - 'layout': self.__layout, + 'layout': self.__sysroots.get_unique_key(), 'root-read-only': self.__root_read_only } @@ -201,72 +187,11 @@ class ScriptElement(Element): # Setup environment sandbox.set_environment(self.get_environment()) - # Tell the sandbox to mount the install root - directories = {self.__install_root: False} - - # Mark the artifact directories in the layout - for item in self.__layout: - destination = item['destination'] - was_artifact = directories.get(destination, False) - directories[destination] = item['element'] or was_artifact - - for directory, artifact in directories.items(): - # Root does not need to be marked as it is always mounted - # with artifact (unless explicitly marked non-artifact) - if directory != '/': - sandbox.mark_directory(directory, artifact=artifact) + self.__sysroots.configure_sandbox(sandbox, [self.__install_root]) def stage(self, sandbox): - # Stage the elements, and run integration commands where appropriate. - if not self.__layout: - # if no layout set, stage all dependencies into / - for build_dep in self.dependencies(Scope.BUILD, recurse=False): - with self.timed_activity("Staging {} at /" - .format(build_dep.name), silent_nested=True): - build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/") - - with sandbox.batch(SandboxFlags.NONE): - for build_dep in self.dependencies(Scope.BUILD, recurse=False): - with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True): - for dep in build_dep.dependencies(Scope.RUN): - dep.integrate(sandbox) - else: - # If layout, follow its rules. - for item in self.__layout: - - # Skip layout members which dont stage an element - if not item['element']: - continue - - element = self.search(Scope.BUILD, item['element']) - if item['destination'] == '/': - with self.timed_activity("Staging {} at /".format(element.name), - silent_nested=True): - element.stage_dependency_artifacts(sandbox, Scope.RUN) - else: - with self.timed_activity("Staging {} at {}" - .format(element.name, item['destination']), - silent_nested=True): - virtual_dstdir = sandbox.get_virtual_directory() - virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True) - element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination']) - - with sandbox.batch(SandboxFlags.NONE): - for item in self.__layout: - - # Skip layout members which dont stage an element - if not item['element']: - continue - - element = self.search(Scope.BUILD, item['element']) - - # Integration commands can only be run for elements staged to / - if item['destination'] == '/': - with self.timed_activity("Integrating {}".format(element.name), - silent_nested=True): - for dep in element.dependencies(Scope.RUN): - dep.integrate(sandbox) + self.__sysroots.stage(sandbox, self.__stage_all) install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep) sandbox.get_virtual_directory().descend(install_root_path_components, create=True) diff --git a/doc/source/format_declaring.rst b/doc/source/format_declaring.rst index 714e1fa33..13858f783 100644 --- a/doc/source/format_declaring.rst +++ b/doc/source/format_declaring.rst @@ -159,6 +159,64 @@ See :ref:`format_dependencies` for more information on the dependency model. The ``runtime-depends`` configuration is available since :ref:`format version 14 ` +Sysroot'ed dependencies +~~~~~~~~~~~~~~~~~~~~~~~ + +Build elements and script elements can support sysroot'ed +dependencies. Sysroot'ed dependencies are intended for bootstraping +base systems or cross-compiling. + +Because sysroot'ed dependencies are plugin specific, they are defined +within the plugin configuration node. + +.. code:: yaml + + config: + # Specify some sysroot'ed dependencies + sysroots: + - path: /sysroot + depends: + - element1.bst + - element2.bst + +During build, or initialization of build shell, sysroot'ed build +dependencies will be staged in the given sysroot path instead of '/' +together with the runtime dependencies of those sysroot'ed build +dependencies. + +It is possible to end up with indirect runtime dependencies in +different sysroots if they are staged from build dependencies with +different sysroots. They will be staged multiple times. + +Sysroot paths only apply to build dependencies. It is not possible to +define runtime dependencies either with ``type: runtime`` or +``runtime-depends``. It is possible to use ``all`` dependencies, but +the sysroot part is only for the build part not the runtime. + +For example: + +.. code:: yaml + + config: + sysroots: + - path: /sysroot + depends: + - element.bst + +is equivalent to: + +.. code:: yaml + + config: + runtime-depends: + - element.bst + sysroots: + - path: /sysroot + build-depends: + - element.bst + +:ref:`Integration commands ` are never executed for +sysroot'ed dependencies. .. _format_sources: diff --git a/tests/sysroot_depends/project/elements/a.bst b/tests/sysroot_depends/project/elements/a.bst new file mode 100644 index 000000000..600aed243 --- /dev/null +++ b/tests/sysroot_depends/project/elements/a.bst @@ -0,0 +1,4 @@ +kind: import +sources: + - kind: local + path: files/a diff --git a/tests/sysroot_depends/project/elements/b.bst b/tests/sysroot_depends/project/elements/b.bst new file mode 100644 index 000000000..ebebf1150 --- /dev/null +++ b/tests/sysroot_depends/project/elements/b.bst @@ -0,0 +1,4 @@ +kind: import +sources: + - kind: local + path: files/b diff --git a/tests/sysroot_depends/project/elements/base.bst b/tests/sysroot_depends/project/elements/base.bst new file mode 100644 index 000000000..3c38c2459 --- /dev/null +++ b/tests/sysroot_depends/project/elements/base.bst @@ -0,0 +1,3 @@ +kind: stack +depends: +- base/base-alpine.bst diff --git a/tests/sysroot_depends/project/elements/base/base-alpine.bst b/tests/sysroot_depends/project/elements/base/base-alpine.bst new file mode 100644 index 000000000..687588f7c --- /dev/null +++ b/tests/sysroot_depends/project/elements/base/base-alpine.bst @@ -0,0 +1,12 @@ +kind: import + +description: | + Alpine Linux base for tests + + Generated using the `tests/integration-tests/base/generate-base.sh` script. + +sources: + - kind: tar + url: alpine:integration-tests-base.v1.x86_64.tar.xz + base-dir: '' + ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 diff --git a/tests/sysroot_depends/project/elements/compose-integration.bst b/tests/sysroot_depends/project/elements/compose-integration.bst new file mode 100644 index 000000000..a6c5ec3f0 --- /dev/null +++ b/tests/sysroot_depends/project/elements/compose-integration.bst @@ -0,0 +1,6 @@ +kind: compose + +sysroots: +- path: /sysroot + build-depends: + - integration.bst diff --git a/tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst b/tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst new file mode 100644 index 000000000..8de79508e --- /dev/null +++ b/tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst @@ -0,0 +1,13 @@ +kind: manual + +build-depends: +- base.bst + +variables: + install-root: "/" + +config: + sysroots: + - path: /other-sysroot + build-depends: + - layer2.bst diff --git a/tests/sysroot_depends/project/elements/compose-layers.bst b/tests/sysroot_depends/project/elements/compose-layers.bst new file mode 100644 index 000000000..498e2fc7f --- /dev/null +++ b/tests/sysroot_depends/project/elements/compose-layers.bst @@ -0,0 +1,4 @@ +kind: compose + +build-depends: +- layer2.bst diff --git a/tests/sysroot_depends/project/elements/integration.bst b/tests/sysroot_depends/project/elements/integration.bst new file mode 100644 index 000000000..e2299b91b --- /dev/null +++ b/tests/sysroot_depends/project/elements/integration.bst @@ -0,0 +1,13 @@ +kind: manual + +depends: +- base.bst + +config: + install-commands: + - echo 0 >"%{install-root}/integrated.txt" + +public: + bst: + integration-commands: + - echo 1 >/integrated.txt diff --git a/tests/sysroot_depends/project/elements/layer1-files.bst b/tests/sysroot_depends/project/elements/layer1-files.bst new file mode 100644 index 000000000..944d800c3 --- /dev/null +++ b/tests/sysroot_depends/project/elements/layer1-files.bst @@ -0,0 +1,4 @@ +kind: import +sources: +- kind: local + path: files/layer1 diff --git a/tests/sysroot_depends/project/elements/layer1.bst b/tests/sysroot_depends/project/elements/layer1.bst new file mode 100644 index 000000000..5d72f78b8 --- /dev/null +++ b/tests/sysroot_depends/project/elements/layer1.bst @@ -0,0 +1,4 @@ +kind: stack + +depends: +- layer1-files.bst diff --git a/tests/sysroot_depends/project/elements/layer2-files.bst b/tests/sysroot_depends/project/elements/layer2-files.bst new file mode 100644 index 000000000..435877d8c --- /dev/null +++ b/tests/sysroot_depends/project/elements/layer2-files.bst @@ -0,0 +1,4 @@ +kind: import +sources: +- kind: local + path: files/layer2 diff --git a/tests/sysroot_depends/project/elements/layer2.bst b/tests/sysroot_depends/project/elements/layer2.bst new file mode 100644 index 000000000..19fa16617 --- /dev/null +++ b/tests/sysroot_depends/project/elements/layer2.bst @@ -0,0 +1,22 @@ +kind: manual + +depends: +- layer2-files.bst + +build-depends: +- base.bst + +config: + sysroots: + - path: /sysroot + depends: + - layer1.bst + + install-commands: + - mkdir -p "%{install-root}" + - | + for file in /*; do + if test -f "${file}"; then + cp "${file}" "%{install-root}" + fi + done diff --git a/tests/sysroot_depends/project/elements/manual-integration-runtime.bst b/tests/sysroot_depends/project/elements/manual-integration-runtime.bst new file mode 100644 index 000000000..0abf89e6d --- /dev/null +++ b/tests/sysroot_depends/project/elements/manual-integration-runtime.bst @@ -0,0 +1,14 @@ +kind: manual + +depends: +- base.bst + +config: + sysroots: + - path: /sysroot + depends: + - integration.bst + + install-commands: + - mkdir -p "%{install-root}" + - echo dummy >"%{install-root}/dummy.txt" diff --git a/tests/sysroot_depends/project/elements/manual-integration.bst b/tests/sysroot_depends/project/elements/manual-integration.bst new file mode 100644 index 000000000..218a7c935 --- /dev/null +++ b/tests/sysroot_depends/project/elements/manual-integration.bst @@ -0,0 +1,15 @@ +kind: manual + +build-depends: +- base.bst + +config: + sysroots: + - path: /sysroot + build-depends: + - integration.bst + + install-commands: + - mkdir -p "%{install-root}/sysroot" + - if test -f /sysroot/integrated.txt; then cp /sysroot/integrated.txt "%{install-root}/sysroot"; fi + - if test -f /integrated.txt; then cp /integrated.txt "%{install-root}"; fi diff --git a/tests/sysroot_depends/project/elements/sysroot-integration.bst b/tests/sysroot_depends/project/elements/sysroot-integration.bst new file mode 100644 index 000000000..0d2e440d4 --- /dev/null +++ b/tests/sysroot_depends/project/elements/sysroot-integration.bst @@ -0,0 +1,10 @@ +kind: manual + +variables: + install-root: "/" + +config: + sysroots: + - path: /sysroot + build-depends: + - integration.bst diff --git a/tests/sysroot_depends/project/elements/target-variable.bst b/tests/sysroot_depends/project/elements/target-variable.bst new file mode 100644 index 000000000..a4568ce20 --- /dev/null +++ b/tests/sysroot_depends/project/elements/target-variable.bst @@ -0,0 +1,14 @@ +kind: manual + +build-depends: +- base.bst + +variables: + mydir: test + install-root: "/path" + +config: + sysroots: + - path: "/path/%{mydir}" + build-depends: + - b.bst diff --git a/tests/sysroot_depends/project/elements/target.bst b/tests/sysroot_depends/project/elements/target.bst new file mode 100644 index 000000000..5c215e44d --- /dev/null +++ b/tests/sysroot_depends/project/elements/target.bst @@ -0,0 +1,14 @@ +kind: manual + +build-depends: +- base.bst +- a.bst + +variables: + install-root: '/' + +config: + sysroots: + - path: /sysroot + build-depends: + - b.bst diff --git a/tests/sysroot_depends/project/files/a/a.txt b/tests/sysroot_depends/project/files/a/a.txt new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/tests/sysroot_depends/project/files/a/a.txt @@ -0,0 +1 @@ +test diff --git a/tests/sysroot_depends/project/files/b/b.txt b/tests/sysroot_depends/project/files/b/b.txt new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/tests/sysroot_depends/project/files/b/b.txt @@ -0,0 +1 @@ +test diff --git a/tests/sysroot_depends/project/files/layer1/1 b/tests/sysroot_depends/project/files/layer1/1 new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/tests/sysroot_depends/project/files/layer1/1 @@ -0,0 +1 @@ +1 diff --git a/tests/sysroot_depends/project/files/layer2/2 b/tests/sysroot_depends/project/files/layer2/2 new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/tests/sysroot_depends/project/files/layer2/2 @@ -0,0 +1 @@ +2 diff --git a/tests/sysroot_depends/project/project.conf b/tests/sysroot_depends/project/project.conf new file mode 100644 index 000000000..e479168d9 --- /dev/null +++ b/tests/sysroot_depends/project/project.conf @@ -0,0 +1,9 @@ +name: test +element-path: elements +aliases: + alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ +options: + linux: + type: bool + description: Whether to expect a linux platform + default: True diff --git a/tests/sysroot_depends/sysroot_depends.py b/tests/sysroot_depends/sysroot_depends.py new file mode 100644 index 000000000..3dabecfc4 --- /dev/null +++ b/tests/sysroot_depends/sysroot_depends.py @@ -0,0 +1,176 @@ +import os +import pytest +from tests.testutils import cli_integration as cli +from tests.testutils.site import IS_LINUX, HAVE_BWRAP + + +# Project directory +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "project", +) + + +@pytest.mark.integration +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') +@pytest.mark.datafiles(DATA_DIR) +def test_sysroot_dependency_smoke_test(datafiles, cli, tmpdir): + "Test simple sysroot use case without integration" + + project = str(datafiles) + checkout = os.path.join(str(tmpdir), 'checkout') + + result = cli.run(project=project, + args=['build', 'target.bst']) + result.assert_success() + + result = cli.run(project=project, + args=['checkout', 'target.bst', checkout]) + result.assert_success() + assert os.path.exists(os.path.join(checkout, 'a.txt')) + assert os.path.exists(os.path.join(checkout, 'sysroot', 'b.txt')) + + +@pytest.mark.integration +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') +@pytest.mark.datafiles(DATA_DIR) +def test_skip_integration_commands_build_element(datafiles, cli, tmpdir): + "Integration commands are not run on sysroots" + + project = str(datafiles) + checkout = os.path.join(str(tmpdir), 'checkout') + + result = cli.run(project=project, + args=['build', 'manual-integration.bst']) + result.assert_success() + + result = cli.run(project=project, + args=['checkout', 'manual-integration.bst', checkout]) + result.assert_success() + + sysroot_integrated = os.path.join(checkout, 'sysroot', 'integrated.txt') + integrated = os.path.join(checkout, 'integrated.txt') + assert os.path.exists(sysroot_integrated) + with open(sysroot_integrated, 'r') as f: + assert f.read() == '0\n' + # We need to make sure that integration command has not been run on / either. + assert not os.path.exists(integrated) + + +@pytest.mark.integration +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') +@pytest.mark.datafiles(DATA_DIR) +def test_sysroot_only_for_build(cli, tmpdir, datafiles): + project = str(datafiles) + checkout = os.path.join(str(tmpdir), 'checkout') + + result = cli.run(project=project, + args=['build', 'compose-layers.bst']) + result.assert_success() + + result = cli.run(project=project, + args=['checkout', 'compose-layers.bst', checkout]) + + result.assert_success() + assert os.path.exists(os.path.join(checkout, '1')) + assert os.path.exists(os.path.join(checkout, '2')) + assert not os.path.exists(os.path.join(checkout, 'sysroot', '1')) + + +@pytest.mark.integration +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') +@pytest.mark.datafiles(DATA_DIR) +def test_sysroot_only_for_build_with_sysroot(cli, tmpdir, datafiles): + project = str(datafiles) + checkout = os.path.join(str(tmpdir), 'checkout') + + result = cli.run(project=project, + args=['build', 'compose-layers-with-sysroot.bst']) + result.assert_success() + + result = cli.run(project=project, + args=['checkout', 'compose-layers-with-sysroot.bst', checkout]) + + result.assert_success() + assert os.path.exists(os.path.join(checkout, 'other-sysroot', '1')) + assert os.path.exists(os.path.join(checkout, 'other-sysroot', '2')) + assert not os.path.exists(os.path.join(checkout, 'sysroot', '1')) + + +@pytest.mark.integration +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') +@pytest.mark.datafiles(DATA_DIR) +def test_shell_no_sysroot(cli, tmpdir, datafiles): + "bst shell does not have sysroots and dependencies are integrated" + + project = str(datafiles) + + result = cli.run(project=project, + args=['build', 'base.bst', 'manual-integration-runtime.bst']) + result.assert_success() + + result = cli.run(project=project, + args=['shell', 'manual-integration-runtime.bst', '--', 'cat', '/integrated.txt']) + result.assert_success() + assert result.output == '1\n' + + result = cli.run(project=project, + args=['shell', 'manual-integration-runtime.bst', '--', 'ls', '/sysroot/integrated.txt']) + assert result.exit_code != 0 + assert result.output == '' + + +@pytest.mark.integration +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') +@pytest.mark.datafiles(DATA_DIR) +def test_shell_build_sysroot(cli, tmpdir, datafiles): + "Build shell should stage build dependencies sysroot'ed non integrated" + + project = str(datafiles) + + result = cli.run(project=project, + args=['build', 'base.bst', 'integration.bst']) + result.assert_success() + + result = cli.run(project=project, + args=['shell', '-b', 'manual-integration.bst', '--', 'cat', '/sysroot/integrated.txt']) + result.assert_success() + assert result.output == '0\n' + + +@pytest.mark.integration +@pytest.mark.datafiles(DATA_DIR) +def test_show_dependencies_only_once(cli, tmpdir, datafiles): + """Dependencies should not show up in status several times when they + are staged with multiple sysroots""" + + project = str(datafiles) + + result = cli.run(project=project, + args=['show', '--format', '%{name}', 'manual-integration.bst']) + result.assert_success() + pipeline = result.output.splitlines() + assert pipeline == ['base/base-alpine.bst', + 'base.bst', + 'integration.bst', + 'manual-integration.bst'] + + +@pytest.mark.integration +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') +@pytest.mark.datafiles(DATA_DIR) +def test_sysroot_path_subst_variable(datafiles, cli, tmpdir): + "Test that variables are expanded in sysroot path" + + project = str(datafiles) + checkout = os.path.join(str(tmpdir), 'checkout') + + result = cli.run(project=project, + args=['build', 'target-variable.bst']) + result.assert_success() + + result = cli.run(project=project, + args=['checkout', 'target-variable.bst', checkout]) + result.assert_success() + + assert os.path.exists(os.path.join(checkout, 'test', 'b.txt')) -- cgit v1.2.1