glupload: Fix fallback from direct dmabuf to dmabuf upload method

In the situation that the direct dmabuf path is chosen, but with an
unsupported texture format, this causes accept to fail rather than
continue and fail at the upload stage. It is also possibly necessary to
reconfigure after falling back from direct to non-direct dmabuf upload
paths.

3 files changed, 72 insertions, 9 deletions

diff --git a/gst-libs/gst/gl/egl/gsteglimage.c b/gst-libs/gst/gl/egl/gsteglimage.c
index 85fbefa6e..13e204dcd 100644
--- a/gst-libs/gst/gl/egl/gsteglimage.c
+++ b/gst-libs/gst/gl/egl/gsteglimage.c
@@ -39,6 +39,7 @@
 #endif
 
 #include "gsteglimage.h"
+#include "gsteglimage_private.h"
 
 #include <string.h>
 
@@ -673,8 +674,20 @@ _drm_direct_fourcc_from_info (GstVideoInfo * info)
   }
 }
 
-static gboolean
-_gst_egl_image_check_dmabuf_direct (GstGLContext * context, int fourcc)
+/*
+ * gst_egl_image_check_dmabuf_direct:
+ * @context: a #GstGLContext (must be an EGL context)
+ * @in_info: a #GstVideoInfo
+ * @target: a #GstGLTextureTarget
+ *
+ * Checks whether the video format specified by the given #GstVideoInfo is a
+ * supported texture format for the given target.
+ *
+ * Returns: %TRUE if the format is supported.
+ */
+gboolean
+gst_egl_image_check_dmabuf_direct (GstGLContext * context,
+    GstVideoInfo * in_info)
 {
   EGLDisplay egl_display = EGL_DEFAULT_DISPLAY;
   GstGLDisplayEGL *display_egl;
@@ -684,6 +697,7 @@ _gst_egl_image_check_dmabuf_direct (GstGLContext * context, int fourcc)
   EGLBoolean *external_only;
   int num_modifiers;
   gboolean ret;
+  int fourcc;
   int i;
 
   EGLBoolean (*gst_eglQueryDmaBufFormatsEXT) (EGLDisplay dpy,
@@ -692,6 +706,10 @@ _gst_egl_image_check_dmabuf_direct (GstGLContext * context, int fourcc)
       int format, int max_modifiers, EGLuint64KHR * modifiers,
       EGLBoolean * external_only, int *num_modifiers);
 
+  fourcc = _drm_direct_fourcc_from_info (in_info);
+  if (fourcc == -1)
+    return FALSE;
+
   gst_eglQueryDmaBufFormatsEXT =
       gst_gl_context_get_proc_address (context, "eglQueryDmaBufFormatsEXT");
   gst_eglQueryDmaBufModifiersEXT =
@@ -812,13 +830,10 @@ gst_egl_image_from_dmabuf_direct (GstGLContext * context,
   guintptr attribs[41];         /* 6 + 10 * 3 + 4 + 1 */
   gint atti = 0;
 
-  fourcc = _drm_direct_fourcc_from_info (in_info);
-  if (fourcc == -1)
-    return NULL;
-
-  if (!_gst_egl_image_check_dmabuf_direct (context, fourcc))
+  if (!gst_egl_image_check_dmabuf_direct (context, in_info))
     return NULL;
 
+  fourcc = _drm_direct_fourcc_from_info (in_info);
   with_modifiers = gst_gl_context_check_feature (context,
       "EGL_EXT_image_dma_buf_import_with_modifiers");
 
diff --git a/gst-libs/gst/gl/egl/gsteglimage_private.h b/gst-libs/gst/gl/egl/gsteglimage_private.h
new file mode 100644
index 000000000..8326bdd05
--- /dev/null
+++ b/gst-libs/gst/gl/egl/gsteglimage_private.h
@@ -0,0 +1,35 @@
+/*
+ * GStreamer
+ * Copyright (C) 2020 Igalia S.L.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _GST_EGL_IMAGE_PRIVATE_H_
+#define _GST_EGL_IMAGE_PRIVATE_H_
+
+#include <gst/gl/gstgl_fwd.h>
+#include <gst/gl/gstglformat.h>
+
+G_BEGIN_DECLS
+
+G_GNUC_INTERNAL
+gboolean                gst_egl_image_check_dmabuf_direct       (GstGLContext * context,
+                                                                 GstVideoInfo * in_info);
+
+G_END_DECLS
+
+#endif /* _GST_EGL_IMAGE_PRIVATE_H_ */
diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index e3e02ebb7..7d19f683a 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -30,6 +30,7 @@
 
 #if GST_GL_HAVE_PLATFORM_EGL
 #include "egl/gsteglimage.h"
+#include "egl/gsteglimage_private.h"
 #include "egl/gstglmemoryegl.h"
 #include "egl/gstglcontext_egl.h"
 #endif
@@ -691,9 +692,12 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
     fd[i] = gst_dmabuf_memory_get_fd (mems[i]);
   }
 
-  if (dmabuf->direct)
+  if (dmabuf->direct) {
+    /* Check if this format is supported by the driver */
     dmabuf->n_mem = 1;
-  else
+    if (!gst_egl_image_check_dmabuf_direct (dmabuf->upload->context, in_info))
+      return FALSE;
+  } else
     dmabuf->n_mem = n_planes;
 
   /* Now create an EGLImage for each dmabufs */
@@ -755,6 +759,15 @@ _dma_buf_upload_perform (gpointer impl, GstBuffer * buffer, GstBuffer ** outbuf)
 {
   struct DmabufUpload *dmabuf = impl;
 
+  /* The direct path sets sinkpad caps to RGBA but this may be incorrect for
+   * the non-direct path, if that path fails to accept. In that case, we need
+   * to reconfigure.
+   */
+  if (!dmabuf->direct &&
+      GST_VIDEO_INFO_FORMAT (&dmabuf->upload->priv->in_info) !=
+      GST_VIDEO_INFO_FORMAT (&dmabuf->out_info))
+    return GST_GL_UPLOAD_RECONFIGURE;
+
   gst_gl_context_thread_add (dmabuf->upload->context,
       (GstGLContextThreadFunc) _dma_buf_upload_perform_gl_thread, dmabuf);