waylandsink: Use memfd_create() when available

This (so-far) Linux- and FreeBSD-only API lets users create file
descriptors purely in memory, without any backing file on the filesystem
and the race condition which could ensue when unlink()ing it.

It also allows seals to be placed on the file, ensuring to every other
process that we won’t be allowed to shrink the contents, potentially
causing a SIGBUS when they try reading it.

This patch is best viewed with the -w option of git log -p.

It is an almost exact copy of Wayland commit
6908c8c85a2e33e5654f64a55cd4f847bf385cae, see
https://gitlab.freedesktop.org/wayland/wayland/merge_requests/4

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1577>

2 files changed, 27 insertions, 11 deletions

diff --git a/ext/wayland/wlshmallocator.c b/ext/wayland/wlshmallocator.c
index 820711372..0a82a35d2 100644
--- a/ext/wayland/wlshmallocator.c
+++ b/ext/wayland/wlshmallocator.c
@@ -49,16 +49,33 @@ gst_wl_shm_allocator_alloc (GstAllocator * allocator, gsize size,
 
   /* TODO: make use of the allocation params, if necessary */
 
-  /* allocate shm pool */
-  snprintf (filename, 1024, "%s/%s-%d-%s", g_get_user_runtime_dir (),
-      "wayland-shm", init++, "XXXXXX");
-
-  fd = g_mkstemp (filename);
-  if (fd < 0) {
-    GST_ERROR_OBJECT (self, "opening temp file %s failed: %s", filename,
-        strerror (errno));
-    return NULL;
+#ifdef HAVE_MEMFD_CREATE
+  fd = memfd_create ("gst-wayland-shm", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+  if (fd >= 0) {
+    /* We can add this seal before calling posix_fallocate(), as
+     * the file is currently zero-sized anyway.
+     *
+     * There is also no need to check for the return value, we
+     * couldn't do anything with it anyway.
+     */
+    fcntl (fd, F_ADD_SEALS, F_SEAL_SHRINK);
+  } else
+#endif
+  {
+    /* allocate shm pool */
+    snprintf (filename, 1024, "%s/%s-%d-%s", g_get_user_runtime_dir (),
+        "wayland-shm", init++, "XXXXXX");
+
+    fd = g_mkstemp (filename);
+    if (fd < 0) {
+      GST_ERROR_OBJECT (self, "opening temp file %s failed: %s", filename,
+          strerror (errno));
+      return NULL;
+    }
+
+    unlink (filename);
   }
+
   if (ftruncate (fd, size) < 0) {
     GST_ERROR_OBJECT (self, "ftruncate failed: %s", strerror (errno));
     close (fd);
@@ -84,8 +101,6 @@ gst_wl_shm_allocator_alloc (GstAllocator * allocator, gsize size,
    * need it to release the miniobject lock */
   gst_memory_unmap (mem, &info);
 
-  unlink (filename);
-
   return mem;
 }
 
diff --git a/meson.build b/meson.build
index d0d1a3fc3..631ad475b 100644
--- a/meson.build
+++ b/meson.build
@@ -158,6 +158,7 @@ check_functions = [
   ['HAVE_DCGETTEXT', 'dcgettext'],
   ['HAVE_GETPAGESIZE', 'getpagesize'],
   ['HAVE_GMTIME_R', 'gmtime_r'],
+  ['HAVE_MEMFD_CREATE', 'memfd_create'],
   ['HAVE_MMAP', 'mmap'],
   ['HAVE_PIPE2', 'pipe2'],
   ['HAVE_GETRUSAGE', 'getrusage', '#include<sys/resource.h>'],