From 61cc93d57d7e2c4780cf23741e240cc56a2a5702 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Tue, 8 Dec 2020 16:52:44 +0900 Subject: plugins/elements/stack.py: Require all dependencies be build & run. Stack elements cannot be build-only dependencies, as this would defeat the purpose of using stack elements in order to directly build-depend on them. Stack element dependencies must all be built in order to build depend on them, and as such we gain no build parallelism by allowing runtime-only dependencies on stack elements. Declaring a runtime-only dependency on a stack element as a whole might still be useful, but still requires the entire stack to be built at the time we need that stack. Instead, it is more useful to ensure that a stack element is a logical group of all dependencies, including runtime dependencies, such that we can guarantee cache key alignment with all stack dependencies. This allows for stronger reliability in commands such as `bst artifact checkout`, which can now reliably download and checkout a fully built stack as a result, without any uncertainty about possible runtime-only dependencies which might exist in the project where that artifact was created. This consequently closes #1075 This also fixes the following tests such that the no longer require build-depends or runtime-depends to work in stack elements: * tests/frontend/default_target.py: Was not necessary to check results of show, these stacks were set to runtime-depends so that they would have the same buildable state as their dependencies when shown. * tests/format/dependencies.py: tests/frontend/pull.py, test/frontend/show.py, tests/integration/compose.py: These tests were using specific build/runtime dependencies in stacks, but for no particular reason. --- src/buildstream/plugins/elements/stack.py | 95 ++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/buildstream/plugins/elements/stack.py b/src/buildstream/plugins/elements/stack.py index b15f67073..bd914ed0a 100644 --- a/src/buildstream/plugins/elements/stack.py +++ b/src/buildstream/plugins/elements/stack.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2016 Codethink Limited +# Copyright (C) 2020 Codethink Limited # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,9 +22,85 @@ stack - Symbolic Element for dependency grouping ================================================ Stack elements are simply a symbolic element used for representing a logical group of elements. + +All dependencies declared in stack elements must always be both +:ref:`build and runtime dependencies `. + +**Example:** + +.. code:: yaml + + kind: stack + + # Declare all of your dependencies in the `depends` list. + depends: + - libc.bst + - coreutils.bst + +.. note:: + + Unlike other elements, whose cache keys are a unique identifier + of the contents of the artifacts they produce, stack elements do + not produce any artifact content. Instead, the cache key of an artifact + is a unique identifier for the assembly of its own dependencies. + + +Using intermediate stacks +------------------------- +Using a stack element at intermediate levels of your build graph +allows you to abstract away some parts of your project into logical +subsystems which elements can more conveniently depend on as a whole. + +In addition to the added convenience, it will allow you to more +easily change the implementation of a subsystem later on, without needing +to update many reverse dependencies to depend on new elements, or even +allow you to conditionally implement a subsystem with various implementations +depending on what :ref:`project options ` were specified at +build time. + + +Using toplevel stacks +--------------------- +Stack elements can also be useful as toplevel targets in your build graph +to simply indicate all of the components which need to be built for a given +system to be complete, or for your integration pipeline to be successful. + + +Checking out and deploying toplevel stacks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In case that your software is built remotely, it is possible to checkout +the built content of a stack on your own machine for the purposes of +inspection or further deployment. + +To accomplish this, you will need to know the cache key of the stack element +which was built remotely, possibly by inspecting the remote build log or by +deriving it with an equally configured BuildStream project, and you will +need read access to the artifact cache server which the build was uploaded to, +this should be configured in your :ref:`user configuration file `. + +You can then checkout the remotely built stack using the +:ref:`bst artifact checkout ` command and providing +it with the :ref:`artifact name `: + +**Example:** + +.. code:: shell + + bst artifact checkout --deps build --pull --integrate \\ + --directory `pwd`/checkout \\ + project/stack/788da21e7c1b5818b7e7b60f7eb75841057ff7e45d362cc223336c606fe47f27 + +.. note:: + + It is possible to checkout other elements in the same way, however stack + elements are uniquely suited to this purpose, as they cannot have + :ref:`runtime only dependencies `, and consequently + their cache keys are always a unique representation of their collective + dependencies. """ -from buildstream import Element +from buildstream import Element, ElementError +from buildstream.types import _Scope # Element implementation for the 'stack' kind. @@ -46,7 +122,20 @@ class StackElement(Element): pass def preflight(self): - pass + + # Assert that all dependencies are both build and runtime dependencies. + # + all_deps = list(self._dependencies(_Scope.ALL, recurse=False)) + run_deps = list(self._dependencies(_Scope.RUN, recurse=False)) + build_deps = list(self._dependencies(_Scope.BUILD, recurse=False)) + if any(dep not in run_deps for dep in all_deps) or any(dep not in build_deps for dep in all_deps): + # There is no need to specify the `self` provenance here in preflight() errors, as the base class + # will take care of prefixing these for plugin author convenience. + raise ElementError( + "All dependencies of 'stack' elements must be both build and runtime dependencies", + detail="Make sure you declare all dependencies in the `depends` list, without specifying any `type`.", + reason="stack-requires-build-and-run", + ) def get_unique_key(self): # We do not add anything to the build, only our dependencies -- cgit v1.2.1