doc/examples/filtering: Added a new user guide entry about filter elements

15 files changed, 433 insertions, 0 deletions

diff --git a/doc/examples/filtering/elements/base.bst b/doc/examples/filtering/elements/base.bst
new file mode 100644
index 000000000..1b85a9e8c
--- /dev/null
+++ b/doc/examples/filtering/elements/base.bst
@@ -0,0 +1,5 @@
+kind: stack
+description: Base stack
+
+depends:
+- base/alpine.bst
diff --git a/doc/examples/filtering/elements/base/alpine.bst b/doc/examples/filtering/elements/base/alpine.bst
new file mode 100644
index 000000000..433f4773a
--- /dev/null
+++ b/doc/examples/filtering/elements/base/alpine.bst
@@ -0,0 +1,17 @@
+kind: import
+description: |
+
+    Alpine Linux base runtime
+
+sources:
+- kind: tar
+  url: alpine:integration-tests-base.v1.x86_64.tar.xz
+  ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
+
+#
+# Run ldconfig in the libdir before running anything
+#
+public:
+  bst:
+    integration-commands:
+    - ldconfig "%{libdir}"
diff --git a/doc/examples/filtering/elements/hello.bst b/doc/examples/filtering/elements/hello.bst
new file mode 100644
index 000000000..d22de63e8
--- /dev/null
+++ b/doc/examples/filtering/elements/hello.bst
@@ -0,0 +1,28 @@
+kind: manual
+description: |
+
+  The hello application
+
+# Depend on the hello library, or the filtered version
+#
+(?):
+- use_filter == True:
+    depends:
+      - libhello-filtered.bst
+- use_filter == False:
+    depends:
+      - libhello.bst
+
+# Stage the files/hello directory for building
+sources:
+  - kind: local
+    path: files/hello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make PREFIX="%{prefix}"
+
+  install-commands:
+  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/filtering/elements/libhello-filtered.bst b/doc/examples/filtering/elements/libhello-filtered.bst
new file mode 100644
index 000000000..a961fb5db
--- /dev/null
+++ b/doc/examples/filtering/elements/libhello-filtered.bst
@@ -0,0 +1,17 @@
+kind: filter
+description: |
+
+  A filtered version of libhello which excludes the defaults
+
+# Specify the build dependency to filter
+build-depends:
+- libhello.bst
+
+# Propagate runtime dependencies
+runtime-depends:
+- base.bst
+
+# Now configure the commands to run
+config:
+  exclude:
+  - defaults
diff --git a/doc/examples/filtering/elements/libhello.bst b/doc/examples/filtering/elements/libhello.bst
new file mode 100644
index 000000000..9bb61abdc
--- /dev/null
+++ b/doc/examples/filtering/elements/libhello.bst
@@ -0,0 +1,31 @@
+kind: manual
+description: |
+
+  The libhello library
+
+# Depend on the base system
+depends:
+- base.bst
+
+# Stage the files/libhello directory for building
+sources:
+  - kind: local
+    path: files/libhello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make PREFIX="%{prefix}"
+
+  install-commands:
+  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
+
+public:
+  bst:
+    # Define a split domain which captures the defaults
+    # which this library installs into %{datadir}
+    #
+    split-rules:
+      defaults:
+      - "%{datadir}/libhello/default-person.txt"
diff --git a/doc/examples/filtering/files/hello/Makefile b/doc/examples/filtering/files/hello/Makefile
new file mode 100644
index 000000000..74d776ff4
--- /dev/null
+++ b/doc/examples/filtering/files/hello/Makefile
@@ -0,0 +1,16 @@
+# Sample makefile for hello.c
+#
+.PHONY: all install
+
+all: hello
+
+install:
+	install -d ${DESTDIR}${PREFIX}/bin
+	install -m 755 hello ${DESTDIR}${PREFIX}/bin
+
+hello: hello.c
+	extra_flags="";                                               \
+	if [ -f "${PREFIX}/share/libhello/default-person.txt" ]; then \
+	  extra_flags=-DDEFAULT_PERSON="\"$$(cat ${PREFIX}/share/libhello/default-person.txt)\""; \
+	fi;                                                           \
+	$(CC) $< -o $@ $${extra_flags} -Wall -lhello
diff --git a/doc/examples/filtering/files/hello/hello.c b/doc/examples/filtering/files/hello/hello.c
new file mode 100644
index 000000000..338baf797
--- /dev/null
+++ b/doc/examples/filtering/files/hello/hello.c
@@ -0,0 +1,25 @@
+/*
+ * hello.c - Simple hello program
+ */
+#include <stdio.h>
+#include <libhello.h>
+
+int main(int argc, char *argv[])
+{
+  const char *person = NULL;
+
+  if (argc > 1)
+    person = argv[1];
+
+  if (person)
+    hello(person);
+  else {
+#ifdef DEFAULT_PERSON
+    hello(DEFAULT_PERSON);
+#else
+    hello("stranger");
+#endif
+  }
+
+  return 0;
+}
diff --git a/doc/examples/filtering/files/libhello/Makefile b/doc/examples/filtering/files/libhello/Makefile
new file mode 100644
index 000000000..ba1f5b6c9
--- /dev/null
+++ b/doc/examples/filtering/files/libhello/Makefile
@@ -0,0 +1,19 @@
+# Sample makefile for hello library
+#
+.PHONY: all install
+
+all: libhello.so
+
+install:
+	install -d ${DESTDIR}${PREFIX}/lib
+	install -d ${DESTDIR}${PREFIX}/include
+	install -d ${DESTDIR}${PREFIX}/share/libhello
+	install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib
+	install -m 644 libhello.h ${DESTDIR}${PREFIX}/include
+	install -m 644 default-person.txt ${DESTDIR}${PREFIX}/share/libhello
+
+%.o: %.c %.h
+	$(CC) -c $< -o $@ -Wall
+
+libhello.so: libhello.o
+	$(CC) -shared -o $@ $<
diff --git a/doc/examples/filtering/files/libhello/default-person.txt b/doc/examples/filtering/files/libhello/default-person.txt
new file mode 100644
index 000000000..ade9d8887
--- /dev/null
+++ b/doc/examples/filtering/files/libhello/default-person.txt
@@ -0,0 +1 @@
+Sophia
diff --git a/doc/examples/filtering/files/libhello/libhello.c b/doc/examples/filtering/files/libhello/libhello.c
new file mode 100644
index 000000000..759b33926
--- /dev/null
+++ b/doc/examples/filtering/files/libhello/libhello.c
@@ -0,0 +1,9 @@
+/*
+ * libhello.c - The hello library
+ */
+#include <stdio.h>
+
+void hello(const char *person)
+{
+  printf("Hello %s\n", person);
+}
diff --git a/doc/examples/filtering/files/libhello/libhello.h b/doc/examples/filtering/files/libhello/libhello.h
new file mode 100644
index 000000000..f714f3659
--- /dev/null
+++ b/doc/examples/filtering/files/libhello/libhello.h
@@ -0,0 +1,8 @@
+/*
+ * libhello.h - The hello library
+ */
+
+/*
+ * A function to say hello to @person
+ */
+void hello(const char *person);
diff --git a/doc/examples/filtering/project.conf b/doc/examples/filtering/project.conf
new file mode 100644
index 000000000..9a01a7515
--- /dev/null
+++ b/doc/examples/filtering/project.conf
@@ -0,0 +1,20 @@
+# Unique project name
+name: filtering
+
+# Required BuildStream format version
+format-version: 18
+
+# Subdirectory where elements are stored
+element-path: elements
+
+# Define an alias for our alpine tarball
+aliases:
+  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
+
+# Use an option to decide if we should use the filter
+#
+options:
+  use_filter:
+    type: bool
+    description: Whether to use a filter on the libhello.bst element
+    default: False
diff --git a/doc/sessions/filtering.run b/doc/sessions/filtering.run
new file mode 100644
index 000000000..2ab3d1acd
--- /dev/null
+++ b/doc/sessions/filtering.run
@@ -0,0 +1,33 @@
+
+commands:
+# Make it fetch first
+- directory: ../examples/filtering
+  command: source fetch hello.bst
+
+# Capture a build output
+- directory: ../examples/filtering
+  command: --option use_filter False build hello.bst
+
+# Capture a build output
+- directory: ../examples/filtering
+  command: --option use_filter True build hello.bst
+
+# Capture list-contents output
+- directory: ../examples/filtering
+  output: ../source/sessions/filtering-list-contents-libhello.html
+  command: artifact list-contents libhello.bst
+
+# Capture list-contents output
+- directory: ../examples/filtering
+  output: ../source/sessions/filtering-list-contents-libhello-filtered.html
+  command: artifact list-contents libhello-filtered.bst
+
+# Capture shell output
+- directory: ../examples/filtering
+  output: ../source/sessions/filtering-shell-without-filter.html
+  command: --option use_filter False shell hello.bst -- hello
+
+# Capture shell output
+- directory: ../examples/filtering
+  output: ../source/sessions/filtering-shell-with-filter.html
+  command: --option use_filter True shell hello.bst -- hello
diff --git a/doc/source/handling-files/filtering.rst b/doc/source/handling-files/filtering.rst
new file mode 100644
index 000000000..27cdfe679
--- /dev/null
+++ b/doc/source/handling-files/filtering.rst
@@ -0,0 +1,203 @@
+
+
+.. _handling_files_filtering:
+
+Filtering
+=========
+In this chapter we will explore how to *filter* the files in an artifact
+using the :mod:`filter <elements.filter>` element, such that an element might
+depend on a subset of the files provided by a filtered element.
+
+.. note::
+
+   This example is distributed with BuildStream
+   in the `doc/examples/filtering
+   <https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/filtering>`_
+   subdirectory.
+
+
+Overview
+--------
+In some cases, it can be useful to depend on a *subset* of the files of an
+element, without depending on the entire element.
+
+One scenario where filtering can be useful, is when you have an element which
+will build differently depending on what is present in the system where it is
+building. In an edge case where a module fails to offer configure time options to
+disable an unwanted feature or behavior in the build, you might use
+:mod:`filter <elements.filter>` elements to ensure that special header files or
+pkg-config files are *filtered out* from the system at build time, such that
+the unwanted behavior cannot be built.
+
+In many ways, a :mod:`filter <elements.filter>` element is like a
+:mod:`compose <elements.compose>` element, except that it operates on a single
+:ref:`build dependency <format_build_depends>`, without compositing the filtered
+element with its :ref:`runtime dependencies <format_runtime_depends>`.
+
+.. tip::
+
+   The :mod:`filter <elements.filter>` element is special in the sense
+   that it acts as a *window* into it's primary
+   :ref:`build dependency <format_build_depends>`.
+
+   As such, :ref:`opening a workspace <invoking_workspace_open>` on a
+   :mod:`filter <elements.filter>` element will result in opening a
+   workspace on the element which it filters. Any other workspace
+   commands will also be forwarded directly to the filtered element.
+
+
+Project structure
+-----------------
+This example again expands on the example presenting in the chapter about
+:ref:`integration commands <tutorial_integration_commands>`. In this case
+we will modify ``libhello.bst`` such that it produces a new file which,
+if present, will affect the behavior of it's reverse dependency ``hello.bst``.
+
+Let's first take a look at how the sources have changed.
+
+
+``files/hello/Makefile``
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/filtering/files/hello/Makefile
+   :language: Makefile
+
+Now we have our Makefile discovering the system defined default
+person to say hello to.
+
+
+``files/hello/hello.c``
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/filtering/files/hello/hello.c
+   :language: c
+
+If this program has been given a ``DEFAULT_PERSON``, then it will
+say hello to that person in the absence of any argument.
+
+
+``project.conf``
+~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/filtering/project.conf
+   :language: yaml
+
+Here, we've added a :ref:`project option <project_options>` to decide
+whether to use the :mod:`filter <elements.filter>` element or not.
+
+This is merely for brevity, so that we can demonstrate the behavior
+of depending on the filtered element without defining two separate versions
+of the ``hello.bst`` element.
+
+
+``elements/libhello.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/filtering/elements/libhello.bst
+   :language: yaml
+
+We've added some :ref:`split rules <public_split_rules>` here to declare
+a new *split domain* named ``defaults``, and we've added the new
+``default-person.txt`` file to this *domain*.
+
+
+``elements/libhello-filtered.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/filtering/elements/libhello-filtered.bst
+   :language: yaml
+
+And we've added a new :mod:`filter <elements.filter>` element to the project
+which uses the ``exclude`` option of the filter configuration.
+
+This is essentially a statement that any files mentioned in the the
+``defaults`` *domain* of the ``libhello.bst`` element should be excluded from
+the resulting artifact.
+
+.. important::
+
+   Notice that you need to explicitly declare any
+   :ref:`runtime dependencies <format_runtime_depends>` which are required by the
+   resulting artifact of a :mod:`filter <elements.filter>` element, as runtime
+   dependencies of the build dependency are not transient.
+
+
+``elements/hello.bst``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/filtering/elements/hello.bst
+   :language: yaml
+
+Here we've merely added a :ref:`conditional statement <format_directives_conditional>`
+which allows us to test the ``hello.bst`` element depending on the filtered
+version of the library, or the unfiltered version.
+
+
+Using the project
+-----------------
+Let's just skip over building the ``hello.bst`` element with the
+``use_filter`` option both ``True`` and ``False``, these elements
+are easily built with :ref:`bst build <invoking_build>` as such:
+
+.. code:: shell
+
+   bst --option use_filter True build hello.bst
+   bst --option use_filter False build hello.bst
+
+
+Observing the artifacts
+~~~~~~~~~~~~~~~~~~~~~~~
+Let's take a look at the built artifacts.
+
+
+``libhello.bst``
+''''''''''''''''
+
+.. raw:: html
+   :file: ../sessions/filtering-list-contents-libhello.html
+
+Here we can see the full content of the ``libhello.bst`` artifact.
+
+
+``libhello-filtered.bst``
+'''''''''''''''''''''''''
+
+.. raw:: html
+   :file: ../sessions/filtering-list-contents-libhello-filtered.html
+
+Here we can see that the ``default-person.txt`` file has been filtered
+out of the ``libhello.bst`` artifact when creating the ``libhello-filtered.bst``
+artifact.
+
+
+Running hello.bst
+~~~~~~~~~~~~~~~~~
+Now if we run the program built by ``hello.bst`` in either build
+modes, we can observe the expected behavior.
+
+
+Run ``hello.bst`` built directly against ``libhello.bst``
+'''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+
+.. raw:: html
+   :file: ../sessions/filtering-shell-without-filter.html
+
+Here we can see that the hello world program is using the system
+configured default person to say hello to.
+
+
+Run ``hello.bst`` built against ``libhello-filtered.bst``
+'''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+
+.. raw:: html
+   :file: ../sessions/filtering-shell-with-filter.html
+
+And now we're reverting to the behavior we have when no
+system configured default person was installed at build time.
+
+
+Summary
+-------
+In this chapter, we've introduced the :mod:`filter <elements.filter>`
+element which allows one to filter the output of an element and
+effectively create a dependency on a subset of an element's files.
diff --git a/doc/source/using_handling_files.rst b/doc/source/using_handling_files.rst
index 8951764e0..58e8a75a2 100644
--- a/doc/source/using_handling_files.rst
+++ b/doc/source/using_handling_files.rst
@@ -8,3 +8,4 @@ handling of files.
    :maxdepth: 1
 
    handling-files/composition.rst
+   handling-files/filtering.rst