summaryrefslogtreecommitdiff
path: root/doc
diff options
context:
space:
mode:
authorTristan Van Berkom <tristan.vanberkom@codethink.co.uk>2020-04-17 20:48:43 +0900
committerTristan Van Berkom <tristan.vanberkom@codethink.co.uk>2020-04-17 20:58:24 +0900
commit98ca2be8d935378885878549758d4a3a90b1824a (patch)
treef8bd4f3e6f6c10d9bb1f8f273f7c2f1ecb2628d1 /doc
parent6b9619b0cbb69310f2543605500a4b82e315738f (diff)
downloadbuildstream-98ca2be8d935378885878549758d4a3a90b1824a.tar.gz
doc/examples/overlaps: Added new user guide entry for overlapping files
Diffstat (limited to 'doc')
-rw-r--r--doc/examples/overlaps/elements/base.bst5
-rw-r--r--doc/examples/overlaps/elements/base/alpine.bst31
-rw-r--r--doc/examples/overlaps/elements/hello.bst22
-rw-r--r--doc/examples/overlaps/elements/libhello.bst22
-rw-r--r--doc/examples/overlaps/elements/runtime-only.bst23
-rw-r--r--doc/examples/overlaps/files/hello/Makefile14
-rw-r--r--doc/examples/overlaps/files/hello/hello.c20
-rw-r--r--doc/examples/overlaps/files/hello/hello.txt1
-rw-r--r--doc/examples/overlaps/files/libhello/Makefile19
-rw-r--r--doc/examples/overlaps/files/libhello/hello.txt1
-rw-r--r--doc/examples/overlaps/files/libhello/libhello.c9
-rw-r--r--doc/examples/overlaps/files/libhello/libhello.h8
-rw-r--r--doc/examples/overlaps/project.conf12
-rw-r--r--doc/sessions/overlaps.run11
-rw-r--r--doc/source/handling-files/overlaps.rst145
-rw-r--r--doc/source/using_handling_files.rst1
16 files changed, 344 insertions, 0 deletions
diff --git a/doc/examples/overlaps/elements/base.bst b/doc/examples/overlaps/elements/base.bst
new file mode 100644
index 000000000..1b85a9e8c
--- /dev/null
+++ b/doc/examples/overlaps/elements/base.bst
@@ -0,0 +1,5 @@
+kind: stack
+description: Base stack
+
+depends:
+- base/alpine.bst
diff --git a/doc/examples/overlaps/elements/base/alpine.bst b/doc/examples/overlaps/elements/base/alpine.bst
new file mode 100644
index 000000000..be89f656f
--- /dev/null
+++ b/doc/examples/overlaps/elements/base/alpine.bst
@@ -0,0 +1,31 @@
+kind: import
+description: |
+
+ Alpine Linux base runtime
+
+sources:
+- kind: tar
+ url: alpine:integration-tests-base.v1.x86_64.tar.xz
+ ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
+
+public:
+ bst:
+ #
+ # Run ldconfig in the libdir before running anything
+ #
+ integration-commands:
+ - ldconfig "%{libdir}"
+
+ #
+ # Extend the runtime split-rule domain for this element,
+ # such that we capture the runtime linker.
+ #
+ # There are various other things provided by this runtime
+ # such as tooling in /bin and an installation of python
+ # and perl, but we'll overlook these for the sake of
+ # this example.
+ #
+ split-rules:
+ runtime:
+ (>):
+ - "/lib/ld*.so*"
diff --git a/doc/examples/overlaps/elements/hello.bst b/doc/examples/overlaps/elements/hello.bst
new file mode 100644
index 000000000..c90254f10
--- /dev/null
+++ b/doc/examples/overlaps/elements/hello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+ The hello application
+
+# Depend on the hello library
+depends:
+- libhello.bst
+
+# Stage the files/hello directory for building
+sources:
+ - kind: local
+ path: files/hello
+
+# Now configure the commands to run
+config:
+
+ build-commands:
+ - make PREFIX="%{prefix}"
+
+ install-commands:
+ - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/overlaps/elements/libhello.bst b/doc/examples/overlaps/elements/libhello.bst
new file mode 100644
index 000000000..53496c84c
--- /dev/null
+++ b/doc/examples/overlaps/elements/libhello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+ The libhello library
+
+# Depend on the base system
+depends:
+- base.bst
+
+# Stage the files/libhello directory for building
+sources:
+ - kind: local
+ path: files/libhello
+
+# Now configure the commands to run
+config:
+
+ build-commands:
+ - make PREFIX="%{prefix}"
+
+ install-commands:
+ - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/overlaps/elements/runtime-only.bst b/doc/examples/overlaps/elements/runtime-only.bst
new file mode 100644
index 000000000..9885b7396
--- /dev/null
+++ b/doc/examples/overlaps/elements/runtime-only.bst
@@ -0,0 +1,23 @@
+kind: compose
+
+# Dependencies of a compose element cannot be transient,
+# we can only build-depend on the inputs of a composition.
+#
+build-depends:
+- hello.bst
+
+config:
+
+ # Only include files from the runtime domain
+ #
+ include:
+ - runtime
+
+ # Don't include any files which do not match any existing
+ # split rule domains.
+ #
+ include-orphans: False
+
+ # Run integration commands before composition
+ #
+ integrate: True
diff --git a/doc/examples/overlaps/files/hello/Makefile b/doc/examples/overlaps/files/hello/Makefile
new file mode 100644
index 000000000..bbf931df3
--- /dev/null
+++ b/doc/examples/overlaps/files/hello/Makefile
@@ -0,0 +1,14 @@
+# Sample makefile for hello.c
+#
+.PHONY: all install
+
+all: hello
+
+install:
+ install -d ${DESTDIR}${PREFIX}/bin
+ install -d ${DESTDIR}${PREFIX}/share/doc
+ install -m 755 hello ${DESTDIR}${PREFIX}/bin
+ install -m 644 hello.txt ${DESTDIR}${PREFIX}/share/doc
+
+hello: hello.c
+ $(CC) $< -o $@ -Wall -lhello
diff --git a/doc/examples/overlaps/files/hello/hello.c b/doc/examples/overlaps/files/hello/hello.c
new file mode 100644
index 000000000..83e762c29
--- /dev/null
+++ b/doc/examples/overlaps/files/hello/hello.c
@@ -0,0 +1,20 @@
+/*
+ * hello.c - Simple hello program
+ */
+#include <stdio.h>
+#include <libhello.h>
+
+int main(int argc, char *argv[])
+{
+ const char *person = NULL;
+
+ if (argc > 1)
+ person = argv[1];
+
+ if (person)
+ hello(person);
+ else
+ hello("stranger");
+
+ return 0;
+}
diff --git a/doc/examples/overlaps/files/hello/hello.txt b/doc/examples/overlaps/files/hello/hello.txt
new file mode 100644
index 000000000..9be06bf35
--- /dev/null
+++ b/doc/examples/overlaps/files/hello/hello.txt
@@ -0,0 +1 @@
+This is the documentation about the hello program.
diff --git a/doc/examples/overlaps/files/libhello/Makefile b/doc/examples/overlaps/files/libhello/Makefile
new file mode 100644
index 000000000..49e5a071a
--- /dev/null
+++ b/doc/examples/overlaps/files/libhello/Makefile
@@ -0,0 +1,19 @@
+# Sample makefile for hello library
+#
+.PHONY: all install
+
+all: libhello.so
+
+install:
+ install -d ${DESTDIR}${PREFIX}/lib
+ install -d ${DESTDIR}${PREFIX}/include
+ install -d ${DESTDIR}${PREFIX}/share/doc
+ install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib
+ install -m 644 libhello.h ${DESTDIR}${PREFIX}/include
+ install -m 644 hello.txt ${DESTDIR}${PREFIX}/share/doc
+
+%.o: %.c %.h
+ $(CC) -c $< -o $@ -Wall
+
+libhello.so: libhello.o
+ $(CC) -shared -o $@ $<
diff --git a/doc/examples/overlaps/files/libhello/hello.txt b/doc/examples/overlaps/files/libhello/hello.txt
new file mode 100644
index 000000000..b97348cb4
--- /dev/null
+++ b/doc/examples/overlaps/files/libhello/hello.txt
@@ -0,0 +1 @@
+This is the documentation about the hello library.
diff --git a/doc/examples/overlaps/files/libhello/libhello.c b/doc/examples/overlaps/files/libhello/libhello.c
new file mode 100644
index 000000000..759b33926
--- /dev/null
+++ b/doc/examples/overlaps/files/libhello/libhello.c
@@ -0,0 +1,9 @@
+/*
+ * libhello.c - The hello library
+ */
+#include <stdio.h>
+
+void hello(const char *person)
+{
+ printf("Hello %s\n", person);
+}
diff --git a/doc/examples/overlaps/files/libhello/libhello.h b/doc/examples/overlaps/files/libhello/libhello.h
new file mode 100644
index 000000000..f714f3659
--- /dev/null
+++ b/doc/examples/overlaps/files/libhello/libhello.h
@@ -0,0 +1,8 @@
+/*
+ * libhello.h - The hello library
+ */
+
+/*
+ * A function to say hello to @person
+ */
+void hello(const char *person);
diff --git a/doc/examples/overlaps/project.conf b/doc/examples/overlaps/project.conf
new file mode 100644
index 000000000..297077220
--- /dev/null
+++ b/doc/examples/overlaps/project.conf
@@ -0,0 +1,12 @@
+# Unique project name
+name: overlaps
+
+# Required BuildStream format version
+format-version: 18
+
+# Subdirectory where elements are stored
+element-path: elements
+
+# Define an alias for our alpine tarball
+aliases:
+ alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
diff --git a/doc/sessions/overlaps.run b/doc/sessions/overlaps.run
new file mode 100644
index 000000000..96ef4f3bd
--- /dev/null
+++ b/doc/sessions/overlaps.run
@@ -0,0 +1,11 @@
+
+commands:
+
+# Build everything up to `hello.bst`
+- directory: ../examples/overlaps
+ command: build hello.bst
+
+# Capture the build output
+- directory: ../examples/overlaps
+ output: ../source/sessions/overlaps-build.html
+ command: build runtime-only.bst
diff --git a/doc/source/handling-files/overlaps.rst b/doc/source/handling-files/overlaps.rst
new file mode 100644
index 000000000..25456d26e
--- /dev/null
+++ b/doc/source/handling-files/overlaps.rst
@@ -0,0 +1,145 @@
+
+
+.. _handling_files_overlaps:
+
+Overlapping files
+=================
+In this chapter, we will discuss what happens when files from multiple
+element artifacts conflict with eachother, and what strategies we can
+use to resolve these situations.
+
+.. note::
+
+ This example is distributed with BuildStream
+ in the `doc/examples/overlaps
+ <https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/overlaps>`_
+ subdirectory.
+
+
+Overview
+--------
+This project builds on the previous chapter on :ref:`composition <handling_files_composition>`,
+and as such we'll only go over what has changed from there, which is not much.
+
+
+Project structure
+-----------------
+In this example we've just extended the ``libhello.bst`` and the ``hello.bst``
+elements such that they both install an additional file: ``%{docdir}/hello.txt``.
+
+We've updated the following Makefiles:
+
+
+files/libhello/Makefile
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/overlaps/files/libhello/Makefile
+ :language: Makefile
+
+
+files/hello/Makefile
+~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/overlaps/files/hello/Makefile
+ :language: Makefile
+
+
+As you can see, this now presents a conflict of sorts, where multiple
+elements in the pipeline want to install the *same file*.
+
+
+Using the project
+-----------------
+In this chapter, we're only going to present the warning and then
+discuss how to mitigate this situation.
+
+See what happens when we try to build the ``runtime-only.bst``
+:mod:`compose <elements.compose>` element:
+
+.. raw:: html
+ :file: ../sessions/overlaps-build.html
+
+Notice the warning message about the conflicting file, it is there to
+inform the user about which files are *overlapping*, and also which
+elements are being staged in which order.
+
+Note also that BuildStream does not discover the overlap until the
+moment that you build a reverse dependency which will require staging
+of both artifacts.
+
+.. tip::
+
+ The ``overlaps`` warning discussed here can be configured to be
+ a :ref:`fatal warning <configurable_warnings>`. This is useful
+ in the case that you want to be strict about avoiding overlapping
+ files in your project.
+
+
+Mitigating overlapping files
+----------------------------
+Since we recently discussed :ref:`filtering of artifacts <handling_files_filtering>`,
+we should note that it is of course possible to handle this case by
+having ``hello.bst`` depend on a *filtered* version of ``libhello.bst`` with the
+offending file excluded.
+
+However, working with :mod:`filter <elements.filter>` elements just for the
+sake of handling a conflicting artifact would be quite inconvenient, so we
+have other means.
+
+
+Whitelisting overlapping files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+BuildStream allows explicitly ignoring such errors by adding the files
+to an :ref:`overlap whitelist <public_overlap_whitelist>`, you could achieve
+this in the given example by adding the following to the ``hello.bst`` element:
+
+.. code:: yaml
+
+ public:
+ bst:
+ overlap-whitelist:
+ - |
+ %{docdir}/hello.txt
+
+.. note::
+
+ Note that :func:`glob patterns <buildstream.utils.glob>` are also
+ supported in the whitelist.
+
+
+Artifact munging
+~~~~~~~~~~~~~~~~
+Another way around this situation is the *munge* the artifacts at install
+time such that there is no conflict.
+
+This is the easiest approach in the case that you might want to keep the
+underlying ``%{docdir}/hello.txt`` from ``libhello.bst`` and discard the
+same file from ``hello.bst``.
+
+In this case, we might modify the ``hello.bst`` file so that it's install
+command contain an ``rm`` statement, as such:
+
+.. code:: yaml
+
+ install-commands:
+ - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
+
+ - |
+ # Rid ourselves of the unwanted file at install time
+ rm -f %{install-root}%{docdir}/hello.txt
+
+This would cause later builds of ``runtime-only.bst`` to no longer
+conflict on the given file.
+
+
+Summary
+-------
+In this chapter we've presented a situation where an artifact
+can *conflict* with another artifact by way of providing the
+same files.
+
+We've presented the :ref:`overlap whitelist <public_overlap_whitelist>`
+public data which is the typical solution for silencing the
+error when the outcome is desired, and also presented a strategy
+to deal with cases where you want to keep files from the
+*overlapped* artifact instead.
diff --git a/doc/source/using_handling_files.rst b/doc/source/using_handling_files.rst
index 58e8a75a2..8d2e3567d 100644
--- a/doc/source/using_handling_files.rst
+++ b/doc/source/using_handling_files.rst
@@ -9,3 +9,4 @@ handling of files.
handling-files/composition.rst
handling-files/filtering.rst
+ handling-files/overlaps.rst