doc/examples/overlaps: Added new user guide entry for overlapping files

16 files changed, 344 insertions, 0 deletions

diff --git a/doc/examples/overlaps/elements/base.bst b/doc/examples/overlaps/elements/base.bst
new file mode 100644
index 000000000..1b85a9e8c
--- /dev/null
+++ b/doc/examples/overlaps/elements/base.bst
@@ -0,0 +1,5 @@
+kind: stack
+description: Base stack
+
+depends:
+- base/alpine.bst
diff --git a/doc/examples/overlaps/elements/base/alpine.bst b/doc/examples/overlaps/elements/base/alpine.bst
new file mode 100644
index 000000000..be89f656f
--- /dev/null
+++ b/doc/examples/overlaps/elements/base/alpine.bst
@@ -0,0 +1,31 @@
+kind: import
+description: |
+
+    Alpine Linux base runtime
+
+sources:
+- kind: tar
+  url: alpine:integration-tests-base.v1.x86_64.tar.xz
+  ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
+
+public:
+  bst:
+    #
+    # Run ldconfig in the libdir before running anything
+    #
+    integration-commands:
+    - ldconfig "%{libdir}"
+
+    #
+    # Extend the runtime split-rule domain for this element,
+    # such that we capture the runtime linker.
+    #
+    # There are various other things provided by this runtime
+    # such as tooling in /bin and an installation of python
+    # and perl, but we'll overlook these for the sake of
+    # this example.
+    #
+    split-rules:
+      runtime:
+        (>):
+        - "/lib/ld*.so*"
diff --git a/doc/examples/overlaps/elements/hello.bst b/doc/examples/overlaps/elements/hello.bst
new file mode 100644
index 000000000..c90254f10
--- /dev/null
+++ b/doc/examples/overlaps/elements/hello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+  The hello application
+
+# Depend on the hello library
+depends:
+- libhello.bst
+
+# Stage the files/hello directory for building
+sources:
+  - kind: local
+    path: files/hello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make PREFIX="%{prefix}"
+
+  install-commands:
+  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/overlaps/elements/libhello.bst b/doc/examples/overlaps/elements/libhello.bst
new file mode 100644
index 000000000..53496c84c
--- /dev/null
+++ b/doc/examples/overlaps/elements/libhello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+  The libhello library
+
+# Depend on the base system
+depends:
+- base.bst
+
+# Stage the files/libhello directory for building
+sources:
+  - kind: local
+    path: files/libhello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make PREFIX="%{prefix}"
+
+  install-commands:
+  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/overlaps/elements/runtime-only.bst b/doc/examples/overlaps/elements/runtime-only.bst
new file mode 100644
index 000000000..9885b7396
--- /dev/null
+++ b/doc/examples/overlaps/elements/runtime-only.bst
@@ -0,0 +1,23 @@
+kind: compose
+
+# Dependencies of a compose element cannot be transient,
+# we can only build-depend on the inputs of a composition.
+#
+build-depends:
+- hello.bst
+
+config:
+
+  # Only include files from the runtime domain
+  #
+  include:
+  - runtime
+
+  # Don't include any files which do not match any existing
+  # split rule domains.
+  #
+  include-orphans: False
+
+  # Run integration commands before composition
+  #
+  integrate: True
diff --git a/doc/examples/overlaps/files/hello/Makefile b/doc/examples/overlaps/files/hello/Makefile
new file mode 100644
index 000000000..bbf931df3
--- /dev/null
+++ b/doc/examples/overlaps/files/hello/Makefile
@@ -0,0 +1,14 @@
+# Sample makefile for hello.c
+#
+.PHONY: all install
+
+all: hello
+
+install:
+	install -d ${DESTDIR}${PREFIX}/bin
+	install -d ${DESTDIR}${PREFIX}/share/doc
+	install -m 755 hello ${DESTDIR}${PREFIX}/bin
+	install -m 644 hello.txt ${DESTDIR}${PREFIX}/share/doc
+
+hello: hello.c
+	$(CC) $< -o $@ -Wall -lhello
diff --git a/doc/examples/overlaps/files/hello/hello.c b/doc/examples/overlaps/files/hello/hello.c
new file mode 100644
index 000000000..83e762c29
--- /dev/null
+++ b/doc/examples/overlaps/files/hello/hello.c
@@ -0,0 +1,20 @@
+/*
+ * hello.c - Simple hello program
+ */
+#include <stdio.h>
+#include <libhello.h>
+
+int main(int argc, char *argv[])
+{
+  const char *person = NULL;
+
+  if (argc > 1)
+    person = argv[1];
+
+  if (person)
+    hello(person);
+  else
+    hello("stranger");
+
+  return 0;
+}
diff --git a/doc/examples/overlaps/files/hello/hello.txt b/doc/examples/overlaps/files/hello/hello.txt
new file mode 100644
index 000000000..9be06bf35
--- /dev/null
+++ b/doc/examples/overlaps/files/hello/hello.txt
@@ -0,0 +1 @@
+This is the documentation about the hello program.
diff --git a/doc/examples/overlaps/files/libhello/Makefile b/doc/examples/overlaps/files/libhello/Makefile
new file mode 100644
index 000000000..49e5a071a
--- /dev/null
+++ b/doc/examples/overlaps/files/libhello/Makefile
@@ -0,0 +1,19 @@
+# Sample makefile for hello library
+#
+.PHONY: all install
+
+all: libhello.so
+
+install:
+	install -d ${DESTDIR}${PREFIX}/lib
+	install -d ${DESTDIR}${PREFIX}/include
+	install -d ${DESTDIR}${PREFIX}/share/doc
+	install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib
+	install -m 644 libhello.h ${DESTDIR}${PREFIX}/include
+	install -m 644 hello.txt ${DESTDIR}${PREFIX}/share/doc
+
+%.o: %.c %.h
+	$(CC) -c $< -o $@ -Wall
+
+libhello.so: libhello.o
+	$(CC) -shared -o $@ $<
diff --git a/doc/examples/overlaps/files/libhello/hello.txt b/doc/examples/overlaps/files/libhello/hello.txt
new file mode 100644
index 000000000..b97348cb4
--- /dev/null
+++ b/doc/examples/overlaps/files/libhello/hello.txt
@@ -0,0 +1 @@
+This is the documentation about the hello library.
diff --git a/doc/examples/overlaps/files/libhello/libhello.c b/doc/examples/overlaps/files/libhello/libhello.c
new file mode 100644
index 000000000..759b33926
--- /dev/null
+++ b/doc/examples/overlaps/files/libhello/libhello.c
@@ -0,0 +1,9 @@
+/*
+ * libhello.c - The hello library
+ */
+#include <stdio.h>
+
+void hello(const char *person)
+{
+  printf("Hello %s\n", person);
+}
diff --git a/doc/examples/overlaps/files/libhello/libhello.h b/doc/examples/overlaps/files/libhello/libhello.h
new file mode 100644
index 000000000..f714f3659
--- /dev/null
+++ b/doc/examples/overlaps/files/libhello/libhello.h
@@ -0,0 +1,8 @@
+/*
+ * libhello.h - The hello library
+ */
+
+/*
+ * A function to say hello to @person
+ */
+void hello(const char *person);
diff --git a/doc/examples/overlaps/project.conf b/doc/examples/overlaps/project.conf
new file mode 100644
index 000000000..297077220
--- /dev/null
+++ b/doc/examples/overlaps/project.conf
@@ -0,0 +1,12 @@
+# Unique project name
+name: overlaps
+
+# Required BuildStream format version
+format-version: 18
+
+# Subdirectory where elements are stored
+element-path: elements
+
+# Define an alias for our alpine tarball
+aliases:
+  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
diff --git a/doc/sessions/overlaps.run b/doc/sessions/overlaps.run
new file mode 100644
index 000000000..96ef4f3bd
--- /dev/null
+++ b/doc/sessions/overlaps.run
@@ -0,0 +1,11 @@
+
+commands:
+
+# Build everything up to `hello.bst`
+- directory: ../examples/overlaps
+  command: build hello.bst
+
+# Capture the build output
+- directory: ../examples/overlaps
+  output: ../source/sessions/overlaps-build.html
+  command: build runtime-only.bst
diff --git a/doc/source/handling-files/overlaps.rst b/doc/source/handling-files/overlaps.rst
new file mode 100644
index 000000000..25456d26e
--- /dev/null
+++ b/doc/source/handling-files/overlaps.rst
@@ -0,0 +1,145 @@
+
+
+.. _handling_files_overlaps:
+
+Overlapping files
+=================
+In this chapter, we will discuss what happens when files from multiple
+element artifacts conflict with eachother, and what strategies we can
+use to resolve these situations.
+
+.. note::
+
+   This example is distributed with BuildStream
+   in the `doc/examples/overlaps
+   <https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/overlaps>`_
+   subdirectory.
+
+
+Overview
+--------
+This project builds on the previous chapter on :ref:`composition <handling_files_composition>`,
+and as such we'll only go over what has changed from there, which is not much.
+
+
+Project structure
+-----------------
+In this example we've just extended the ``libhello.bst`` and the ``hello.bst``
+elements such that they both install an additional file: ``%{docdir}/hello.txt``.
+
+We've updated the following Makefiles:
+
+
+files/libhello/Makefile
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/overlaps/files/libhello/Makefile
+   :language: Makefile
+
+
+files/hello/Makefile
+~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/overlaps/files/hello/Makefile
+   :language: Makefile
+
+
+As you can see, this now presents a conflict of sorts, where multiple
+elements in the pipeline want to install the *same file*.
+
+
+Using the project
+-----------------
+In this chapter, we're only going to present the warning and then
+discuss how to mitigate this situation.
+
+See what happens when we try to build the ``runtime-only.bst``
+:mod:`compose <elements.compose>` element:
+
+.. raw:: html
+   :file: ../sessions/overlaps-build.html
+
+Notice the warning message about the conflicting file, it is there to
+inform the user about which files are *overlapping*, and also which
+elements are being staged in which order.
+
+Note also that BuildStream does not discover the overlap until the
+moment that you build a reverse dependency which will require staging
+of both artifacts.
+
+.. tip::
+
+   The ``overlaps`` warning discussed here can be configured to be
+   a :ref:`fatal warning <configurable_warnings>`. This is useful
+   in the case that you want to be strict about avoiding overlapping
+   files in your project.
+
+
+Mitigating overlapping files
+----------------------------
+Since we recently discussed :ref:`filtering of artifacts <handling_files_filtering>`,
+we should note that it is of course possible to handle this case by
+having ``hello.bst`` depend on a *filtered* version of ``libhello.bst`` with the
+offending file excluded.
+
+However, working with :mod:`filter <elements.filter>` elements just for the
+sake of handling a conflicting artifact would be quite inconvenient, so we
+have other means.
+
+
+Whitelisting overlapping files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+BuildStream allows explicitly ignoring such errors by adding the files
+to an :ref:`overlap whitelist <public_overlap_whitelist>`, you could achieve
+this in the given example by adding the following to the ``hello.bst`` element:
+
+.. code:: yaml
+
+  public:
+    bst:
+      overlap-whitelist:
+      - |
+        %{docdir}/hello.txt
+
+.. note::
+
+   Note that :func:`glob patterns <buildstream.utils.glob>` are also
+   supported in the whitelist.
+
+
+Artifact munging
+~~~~~~~~~~~~~~~~
+Another way around this situation is the *munge* the artifacts at install
+time such that there is no conflict.
+
+This is the easiest approach in the case that you might want to keep the
+underlying ``%{docdir}/hello.txt`` from ``libhello.bst`` and discard the
+same file from ``hello.bst``.
+
+In this case, we might modify the ``hello.bst`` file so that it's install
+command contain an ``rm`` statement, as such:
+
+.. code:: yaml
+
+  install-commands:
+  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
+
+  - |
+    # Rid ourselves of the unwanted file at install time
+    rm -f %{install-root}%{docdir}/hello.txt
+
+This would cause later builds of ``runtime-only.bst`` to no longer
+conflict on the given file.
+
+
+Summary
+-------
+In this chapter we've presented a situation where an artifact
+can *conflict* with another artifact by way of providing the
+same files.
+
+We've presented the :ref:`overlap whitelist <public_overlap_whitelist>`
+public data which is the typical solution for silencing the
+error when the outcome is desired, and also presented a strategy
+to deal with cases where you want to keep files from the
+*overlapped* artifact instead.
diff --git a/doc/source/using_handling_files.rst b/doc/source/using_handling_files.rst
index 58e8a75a2..8d2e3567d 100644
--- a/doc/source/using_handling_files.rst
+++ b/doc/source/using_handling_files.rst
@@ -9,3 +9,4 @@ handling of files.
 
    handling-files/composition.rst
    handling-files/filtering.rst
+   handling-files/overlaps.rst