summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--doc/Makefile3
-rw-r--r--doc/examples/composition/elements/base.bst5
-rw-r--r--doc/examples/composition/elements/base/alpine.bst31
-rw-r--r--doc/examples/composition/elements/hello.bst22
-rw-r--r--doc/examples/composition/elements/libhello.bst22
-rw-r--r--doc/examples/composition/elements/runtime-only.bst23
-rw-r--r--doc/examples/composition/files/hello/Makefile12
-rw-r--r--doc/examples/composition/files/hello/hello.c20
-rw-r--r--doc/examples/composition/files/libhello/Makefile17
-rw-r--r--doc/examples/composition/files/libhello/libhello.c9
-rw-r--r--doc/examples/composition/files/libhello/libhello.h8
-rw-r--r--doc/examples/composition/project.conf12
-rw-r--r--doc/sessions/composition.run20
-rw-r--r--doc/source/format_project.rst2
-rw-r--r--doc/source/handling-files/composition.rst194
-rw-r--r--doc/source/main_using.rst1
-rw-r--r--doc/source/using_handling_files.rst10
17 files changed, 410 insertions, 1 deletions
diff --git a/doc/Makefile b/doc/Makefile
index a7150e6d7..4a9373d34 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -143,11 +143,12 @@ html devhelp: templates sessions badges
$(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \
$(wildcard source/*.rst) \
$(wildcard source/tutorial/*.rst) \
+ $(wildcard source/developing/*.rst) \
+ $(wildcard source/handling-files/*.rst) \
$(wildcard source/junctions/*.rst) \
$(wildcard source/examples/*.rst) \
$(wildcard source/elements/*.rst) \
$(wildcard source/sources/*.rst) \
- $(wildcard source/developing/*.rst) \
$(wildcard source/hacking/*.rst)
@echo
@echo "Build of $@ finished, output: $(CURDIR)/$(BUILDDIR)/$@"
diff --git a/doc/examples/composition/elements/base.bst b/doc/examples/composition/elements/base.bst
new file mode 100644
index 000000000..1b85a9e8c
--- /dev/null
+++ b/doc/examples/composition/elements/base.bst
@@ -0,0 +1,5 @@
+kind: stack
+description: Base stack
+
+depends:
+- base/alpine.bst
diff --git a/doc/examples/composition/elements/base/alpine.bst b/doc/examples/composition/elements/base/alpine.bst
new file mode 100644
index 000000000..be89f656f
--- /dev/null
+++ b/doc/examples/composition/elements/base/alpine.bst
@@ -0,0 +1,31 @@
+kind: import
+description: |
+
+ Alpine Linux base runtime
+
+sources:
+- kind: tar
+ url: alpine:integration-tests-base.v1.x86_64.tar.xz
+ ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
+
+public:
+ bst:
+ #
+ # Run ldconfig in the libdir before running anything
+ #
+ integration-commands:
+ - ldconfig "%{libdir}"
+
+ #
+ # Extend the runtime split-rule domain for this element,
+ # such that we capture the runtime linker.
+ #
+ # There are various other things provided by this runtime
+ # such as tooling in /bin and an installation of python
+ # and perl, but we'll overlook these for the sake of
+ # this example.
+ #
+ split-rules:
+ runtime:
+ (>):
+ - "/lib/ld*.so*"
diff --git a/doc/examples/composition/elements/hello.bst b/doc/examples/composition/elements/hello.bst
new file mode 100644
index 000000000..c90254f10
--- /dev/null
+++ b/doc/examples/composition/elements/hello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+ The hello application
+
+# Depend on the hello library
+depends:
+- libhello.bst
+
+# Stage the files/hello directory for building
+sources:
+ - kind: local
+ path: files/hello
+
+# Now configure the commands to run
+config:
+
+ build-commands:
+ - make PREFIX="%{prefix}"
+
+ install-commands:
+ - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/composition/elements/libhello.bst b/doc/examples/composition/elements/libhello.bst
new file mode 100644
index 000000000..53496c84c
--- /dev/null
+++ b/doc/examples/composition/elements/libhello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+ The libhello library
+
+# Depend on the base system
+depends:
+- base.bst
+
+# Stage the files/libhello directory for building
+sources:
+ - kind: local
+ path: files/libhello
+
+# Now configure the commands to run
+config:
+
+ build-commands:
+ - make PREFIX="%{prefix}"
+
+ install-commands:
+ - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/composition/elements/runtime-only.bst b/doc/examples/composition/elements/runtime-only.bst
new file mode 100644
index 000000000..9885b7396
--- /dev/null
+++ b/doc/examples/composition/elements/runtime-only.bst
@@ -0,0 +1,23 @@
+kind: compose
+
+# Dependencies of a compose element cannot be transient,
+# we can only build-depend on the inputs of a composition.
+#
+build-depends:
+- hello.bst
+
+config:
+
+ # Only include files from the runtime domain
+ #
+ include:
+ - runtime
+
+ # Don't include any files which do not match any existing
+ # split rule domains.
+ #
+ include-orphans: False
+
+ # Run integration commands before composition
+ #
+ integrate: True
diff --git a/doc/examples/composition/files/hello/Makefile b/doc/examples/composition/files/hello/Makefile
new file mode 100644
index 000000000..21471d40f
--- /dev/null
+++ b/doc/examples/composition/files/hello/Makefile
@@ -0,0 +1,12 @@
+# Sample makefile for hello.c
+#
+.PHONY: all install
+
+all: hello
+
+install:
+ install -d ${DESTDIR}${PREFIX}/bin
+ install -m 755 hello ${DESTDIR}${PREFIX}/bin
+
+hello: hello.c
+ $(CC) $< -o $@ -Wall -lhello
diff --git a/doc/examples/composition/files/hello/hello.c b/doc/examples/composition/files/hello/hello.c
new file mode 100644
index 000000000..83e762c29
--- /dev/null
+++ b/doc/examples/composition/files/hello/hello.c
@@ -0,0 +1,20 @@
+/*
+ * hello.c - Simple hello program
+ */
+#include <stdio.h>
+#include <libhello.h>
+
+int main(int argc, char *argv[])
+{
+ const char *person = NULL;
+
+ if (argc > 1)
+ person = argv[1];
+
+ if (person)
+ hello(person);
+ else
+ hello("stranger");
+
+ return 0;
+}
diff --git a/doc/examples/composition/files/libhello/Makefile b/doc/examples/composition/files/libhello/Makefile
new file mode 100644
index 000000000..63ee11069
--- /dev/null
+++ b/doc/examples/composition/files/libhello/Makefile
@@ -0,0 +1,17 @@
+# Sample makefile for hello library
+#
+.PHONY: all install
+
+all: libhello.so
+
+install:
+ install -d ${DESTDIR}${PREFIX}/lib
+ install -d ${DESTDIR}${PREFIX}/include
+ install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib
+ install -m 644 libhello.h ${DESTDIR}${PREFIX}/include
+
+%.o: %.c %.h
+ $(CC) -c $< -o $@ -Wall
+
+libhello.so: libhello.o
+ $(CC) -shared -o $@ $<
diff --git a/doc/examples/composition/files/libhello/libhello.c b/doc/examples/composition/files/libhello/libhello.c
new file mode 100644
index 000000000..759b33926
--- /dev/null
+++ b/doc/examples/composition/files/libhello/libhello.c
@@ -0,0 +1,9 @@
+/*
+ * libhello.c - The hello library
+ */
+#include <stdio.h>
+
+void hello(const char *person)
+{
+ printf("Hello %s\n", person);
+}
diff --git a/doc/examples/composition/files/libhello/libhello.h b/doc/examples/composition/files/libhello/libhello.h
new file mode 100644
index 000000000..f714f3659
--- /dev/null
+++ b/doc/examples/composition/files/libhello/libhello.h
@@ -0,0 +1,8 @@
+/*
+ * libhello.h - The hello library
+ */
+
+/*
+ * A function to say hello to @person
+ */
+void hello(const char *person);
diff --git a/doc/examples/composition/project.conf b/doc/examples/composition/project.conf
new file mode 100644
index 000000000..dc82df72b
--- /dev/null
+++ b/doc/examples/composition/project.conf
@@ -0,0 +1,12 @@
+# Unique project name
+name: composition
+
+# Required BuildStream format version
+format-version: 18
+
+# Subdirectory where elements are stored
+element-path: elements
+
+# Define an alias for our alpine tarball
+aliases:
+ alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
diff --git a/doc/sessions/composition.run b/doc/sessions/composition.run
new file mode 100644
index 000000000..c19a9914e
--- /dev/null
+++ b/doc/sessions/composition.run
@@ -0,0 +1,20 @@
+
+commands:
+# Make it fetch first
+- directory: ../examples/composition
+ command: source fetch hello.bst
+
+# Capture the build output
+- directory: ../examples/composition
+ output: ../source/sessions/composition-build.html
+ command: build runtime-only.bst
+
+# List the contents
+- directory: ../examples/composition
+ output: ../source/sessions/composition-list-contents.html
+ command: artifact list-contents runtime-only.bst
+
+# Run hello world
+- directory: ../examples/composition
+ output: ../source/sessions/composition-shell.html
+ command: shell runtime-only.bst -- hello audience
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index 978750031..549743146 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -779,6 +779,8 @@ the number of jobs for a given build without affecting the resulting
cache key.
+.. _project_split_rules:
+
Split rules
~~~~~~~~~~~
The project wide :ref:`split rules <public_split_rules>` defaults can
diff --git a/doc/source/handling-files/composition.rst b/doc/source/handling-files/composition.rst
new file mode 100644
index 000000000..729ed88ee
--- /dev/null
+++ b/doc/source/handling-files/composition.rst
@@ -0,0 +1,194 @@
+
+
+.. _handling_files_composition:
+
+Composition
+===========
+In this chapter we will explore how to create *compositions* of multiple
+input filesystem trees, using the :mod:`compose <elements.compose>` element.
+
+.. note::
+
+ This example is distributed with BuildStream
+ in the `doc/examples/composition
+ <https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/composition>`_
+ subdirectory.
+
+
+Overview
+--------
+Composing a directory tree based on a set of build dependencies is often
+one of the important steps you might perform in order to create a single artifact
+which can be checked out and deployed.
+
+In order to use the :mod:`compose <elements.compose>` element, it is important
+to first understand the concept of :ref:`split rules <public_split_rules>`, which
+we will cover in this chapter.
+
+
+Introducing split rules
+~~~~~~~~~~~~~~~~~~~~~~~
+The :ref:`split rules <public_split_rules>` of an element declaration denote
+which sets of files in the given element's resulting artifact belong to which
+*domain name*.
+
+The *domains* can then be used in various ways, using plugins which understand
+*split rule domains*.
+
+BuildStream's :ref:`default project configuration <project_builtin_defaults>`
+contains a sensible set of default *split rule domains* for the purpose of
+artifact splitting, they can be overridden in :ref:`your project.conf <project_split_rules>`,
+and finally on a per element basis in the :ref:`public data <public_builtin>`
+of your element declarations.
+
+.. note::
+
+ Projects are free to add additional *split rule domains* on top of the
+ default domains provided by the default project configuration.
+
+ There is nothing wrong with defining split rule domains which *overlap*,
+ possibly capturing some of the same files also captured by another
+ *domain*, however you should be aware of this when later using your
+ split rules with a plugin which processes them, like the
+ :mod:`compose <elements.compose>` element described in this chapter.
+
+
+Example of split rule declaration
+'''''''''''''''''''''''''''''''''
+In an element, you might need to define or extend the ``split-rules``
+in order to capture files in custom locations in a logical *domain*.
+
+Here is an example of how you might use the
+:ref:`list append directive <format_directives_list_append>`
+to append an additional rule to your ``split-rules`` list in order to
+capture additional data files which your application or library might
+want to include in the *runtime domain*:
+
+.. code:: yaml
+
+ # Add our .dat files to the runtime domain
+ public:
+ bst:
+ split-rules:
+ runtime:
+ (>):
+ - |
+ %{datadir}/foo/*.dat
+
+Split rules are absolute paths which denote files within an artifact's root
+directory. The globbing patterns supported in split rules are defined in the
+:func:`reference documentation here <buildstream.utils.glob>`.
+
+.. important::
+
+ Note that because of variable expansion, split rules can often be
+ *resolved differently* for elements which have overridden path
+ related variables, like ``%{prefix}``.
+
+ This usually means that you do not need to explicitly extend or override
+ split rules on a specific element unless your element installs files to
+ special case locations.
+
+
+Project structure
+-----------------
+In this example we expand on the chapter about
+:ref:`integration commands <tutorial_integration_commands>`, so we will
+only discuss the files which are added or changed from that example.
+
+
+``elements/base/alpine.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/composition/elements/base/alpine.bst
+ :language: yaml
+
+Here we have modified the base runtime, so as to specify that for this
+element, we want to also include the runtime linker into the *runtime domain*.
+
+
+``elements/runtime-only.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/composition/elements/runtime-only.bst
+ :language: yaml
+
+As we can see, this :mod:`compose <elements.compose>` element has
+been configured to only include files from the *runtime domain*.
+
+
+Using the project
+-----------------
+Now that we've presented how :ref:`split rules <public_split_rules>`
+work and shown how to use them in the context of this example, lets
+use the :mod:`compose <elements.compose>` element we've created and
+observe the results.
+
+
+Building the project
+~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+ :file: ../sessions/composition-build.html
+
+As you can see in the output, this composition has only a few hundred
+files, but the complete ``alpine.bst`` runtime has several thousand
+files.
+
+
+List the content
+~~~~~~~~~~~~~~~~
+At the risk of this being a long list, let's :ref:`list the
+contents of this artifact <invoking_artifact_list_contents>`
+
+.. raw:: html
+ :file: ../sessions/composition-list-contents.html
+
+Some things to observe here:
+
+* The list does include the ``/usr/bin/hello`` program and
+ also the ``/usr/lib/libhello.so`` shared library.
+
+ These paths are both captured by the default split rules
+ for the *runtime domain*.
+
+* The list does not include the ``/usr/include/libhello.h``
+ header file which was used to compile ``/usr/bin/hello``.
+
+ The header file is not captured by the *runtime domain*
+ by default. It is however captured by the *devel domain*.
+
+* The runtime linker ``/lib/ld-musl-x86_64.so.1``, as this was
+ explicitly added to the *runtime domain* for the ``base/alpine.bst``
+ element which provides this file.
+
+.. tip::
+
+ The reader at this time might want to list the content of
+ other elements built from this project, such as the
+ ``hello.bst`` element by itself, or the ``base/alpine.bst``
+ element.
+
+
+Run the program
+~~~~~~~~~~~~~~~
+Finally, lets just run the program we built.
+
+.. raw:: html
+ :file: ../sessions/composition-shell.html
+
+Here we can see that we at least have the required files to run
+our hello world program, however we would not have if we were
+missing the runtime linker which we added in ``base/alpine.bst``.
+
+
+Summary
+-------
+In this chapter we've gotten familiar with :ref:`split rules <public_split_rules>`
+annotations, and we've learned enough about the :mod:`compose <elements.compose>`
+element such that we can start creating our own compositions using
+*split domains*.
+
+We've also used the :ref:`list append directive <format_directives_list_append>`
+and we are now observing the contents of artifacts using
+:ref:`bst artifact list-contents <invoking_artifact_list_contents>`.
diff --git a/doc/source/main_using.rst b/doc/source/main_using.rst
index 067e7bb16..ddda44a26 100644
--- a/doc/source/main_using.rst
+++ b/doc/source/main_using.rst
@@ -13,6 +13,7 @@ guides and information on user preferences and configuration.
using_tutorial
using_developing
+ using_handling_files
using_junctions
using_config
using_commands
diff --git a/doc/source/using_handling_files.rst b/doc/source/using_handling_files.rst
new file mode 100644
index 000000000..8951764e0
--- /dev/null
+++ b/doc/source/using_handling_files.rst
@@ -0,0 +1,10 @@
+Handling files
+==============
+This section covers the various aspects related to finer grained
+handling of files.
+
+.. toctree::
+ :numbered:
+ :maxdepth: 1
+
+ handling-files/composition.rst