videoaggregator: fix caps query to properly handle alpha formats

Only accept alpha if downstream has alpha as well. It could
theoretically accept alpha unconditionally if blending is
properly implemented for handle it but at the moment this
is a missing feature.

Improves the caps query by also comparing with the template
caps to filter by what the subclass supports.

https://bugzilla.gnome.org/show_bug.cgi?id=754465

2 files changed, 199 insertions, 12 deletions

diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c
index 80b10d445..053e81be6 100644
--- a/gst-libs/gst/video/gstvideoaggregator.c
+++ b/gst-libs/gst/video/gstvideoaggregator.c
@@ -76,6 +76,7 @@ struct _GstVideoAggregatorPadPrivate
   GstClockTime end_time;
 };
 
+
 G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad,
     GST_TYPE_AGGREGATOR_PAD);
 
@@ -429,9 +430,44 @@ struct _GstVideoAggregatorPrivate
   gboolean live;
 };
 
-G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVideoAggregator, gst_videoaggregator,
-    GST_TYPE_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
-        gst_videoaggregator_child_proxy_init));
+/* Can't use the G_DEFINE_TYPE macros because we need the
+ * videoaggregator class in the _init to be able to set
+ * the sink pad non-alpha caps. Using the G_DEFINE_TYPE there
+ * seems to be no way of getting the real class being initialized */
+static void gst_videoaggregator_init (GstVideoAggregator * self,
+    GstVideoAggregatorClass * klass);
+static void gst_videoaggregator_class_init (GstVideoAggregatorClass * klass);
+static gpointer gst_videoaggregator_parent_class = NULL;
+static gint GstVideoAggregator_private_offset;
+
+_G_DEFINE_TYPE_EXTENDED_CLASS_INIT (GstVideoAggregator, gst_videoaggregator);
+
+G_GNUC_UNUSED static inline gpointer
+gst_videoaggregator_get_instance_private (const GstVideoAggregator * self)
+{
+  return (G_STRUCT_MEMBER_P (self, GstVideoAggregator_private_offset));
+}
+
+GType
+gst_videoaggregator_get_type (void)
+{
+  static volatile gsize g_define_type_id_volatile = 0;
+  if (g_once_init_enter (&g_define_type_id_volatile)) {
+    GType g_define_type_id = g_type_register_static_simple (GST_TYPE_AGGREGATOR,
+        g_intern_static_string ("GstVideoAggregator"),
+        sizeof (GstVideoAggregatorClass),
+        (GClassInitFunc) gst_videoaggregator_class_intern_init,
+        sizeof (GstVideoAggregator),
+        (GInstanceInitFunc) gst_videoaggregator_init,
+        (GTypeFlags) G_TYPE_FLAG_ABSTRACT);
+    {
+      G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
+          gst_videoaggregator_child_proxy_init);
+    }
+    g_once_init_leave (&g_define_type_id_volatile, g_define_type_id);
+  }
+  return g_define_type_id_volatile;
+}
 
 static void
 gst_videoaggreagator_find_best_format (GstVideoAggregator * vagg,
@@ -833,27 +869,80 @@ beach:
   return ret;
 }
 
+static gboolean
+gst_videoaggregator_caps_has_alpha (GstCaps * caps)
+{
+  guint size = gst_caps_get_size (caps);
+  guint i;
+
+  for (i = 0; i < size; i++) {
+    GstStructure *s = gst_caps_get_structure (caps, i);
+    const GValue *formats = gst_structure_get_value (s, "format");
+
+    if (formats) {
+      const GstVideoFormatInfo *info;
+
+      if (GST_VALUE_HOLDS_LIST (formats)) {
+        guint list_size = gst_value_list_get_size (formats);
+        guint index;
+
+        for (index = 0; index < list_size; index++) {
+          const GValue *list_item = gst_value_list_get_value (formats, index);
+          info =
+              gst_video_format_get_info (gst_video_format_from_string
+              (g_value_get_string (list_item)));
+          if (GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info))
+            return TRUE;
+        }
+
+      } else if (G_VALUE_HOLDS_STRING (formats)) {
+        info =
+            gst_video_format_get_info (gst_video_format_from_string
+            (g_value_get_string (formats)));
+        if (GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info))
+          return TRUE;
+
+      } else {
+        g_assert_not_reached ();
+        GST_WARNING ("Unexpected type for video 'format' field: %s",
+            G_VALUE_TYPE_NAME (formats));
+      }
+
+    } else {
+      return TRUE;
+    }
+  }
+  return FALSE;
+}
+
 static GstCaps *
 gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg,
     GstCaps * filter)
 {
   GstCaps *srccaps;
-  GstCaps *template_caps;
+  GstCaps *template_caps, *sink_template_caps;
   GstCaps *returned_caps;
   GstStructure *s;
-  gboolean had_current_caps = TRUE;
   gint i, n;
   GstAggregator *agg = GST_AGGREGATOR (vagg);
+  GstPad *srcpad = GST_PAD (agg->srcpad);
+  gboolean has_alpha;
 
-  template_caps = gst_pad_get_pad_template_caps (GST_PAD (agg->srcpad));
+  template_caps = gst_pad_get_pad_template_caps (srcpad);
 
-  srccaps = gst_pad_get_current_caps (GST_PAD (agg->srcpad));
+  GST_DEBUG_OBJECT (pad, "Get caps with filter: %" GST_PTR_FORMAT, filter);
+
+  srccaps = gst_pad_get_current_caps (srcpad);
   if (srccaps == NULL) {
-    had_current_caps = FALSE;
-    srccaps = template_caps;
+    srccaps = gst_pad_peer_query_caps (srcpad, template_caps);
+    GST_DEBUG_OBJECT (pad, "No output caps, using possible formats: %"
+        GST_PTR_FORMAT, srccaps);
+  } else {
+    GST_DEBUG_OBJECT (pad, "Using output caps: %" GST_PTR_FORMAT, srccaps);
   }
 
   srccaps = gst_caps_make_writable (srccaps);
+  has_alpha = gst_videoaggregator_caps_has_alpha (srccaps);
 
   n = gst_caps_get_size (srccaps);
   for (i = 0; i < n; i++) {
@@ -873,8 +962,23 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg,
     returned_caps = srccaps;
   }
 
-  if (had_current_caps)
-    gst_caps_unref (template_caps);
+  if (has_alpha) {
+    sink_template_caps = gst_pad_get_pad_template_caps (pad);
+  } else {
+    GstVideoAggregatorClass *klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg);
+    sink_template_caps = gst_caps_ref (klass->sink_non_alpha_caps);
+  }
+
+  {
+    GstCaps *intersect = gst_caps_intersect (returned_caps, sink_template_caps);
+    gst_caps_unref (returned_caps);
+    returned_caps = intersect;
+  }
+
+  gst_caps_unref (template_caps);
+  gst_caps_unref (sink_template_caps);
+
+  GST_DEBUG_OBJECT (pad, "Returning caps: %" GST_PTR_FORMAT, returned_caps);
 
   return returned_caps;
 }
@@ -1974,9 +2078,83 @@ gst_videoaggregator_class_init (GstVideoAggregatorClass * klass)
   g_type_class_ref (GST_TYPE_VIDEO_AGGREGATOR_PAD);
 }
 
+static inline GstCaps *
+_get_non_alpha_caps_from_template (GstVideoAggregatorClass * klass)
+{
+  GstCaps *result;
+  GstCaps *templatecaps;
+  guint i, size;
+
+  templatecaps =
+      gst_pad_template_get_caps (gst_element_class_get_pad_template
+      (GST_ELEMENT_CLASS (klass), "sink_%u"));
+
+  size = gst_caps_get_size (templatecaps);
+  result = gst_caps_new_empty ();
+  for (i = 0; i < size; i++) {
+    GstStructure *s = gst_caps_get_structure (templatecaps, i);
+    const GValue *formats = gst_structure_get_value (s, "format");
+    GValue new_formats = { 0, };
+    gboolean has_format = FALSE;
+
+    /* FIXME what to do if formats are missing? */
+    if (formats) {
+      const GstVideoFormatInfo *info;
+
+      if (GST_VALUE_HOLDS_LIST (formats)) {
+        guint list_size = gst_value_list_get_size (formats);
+        guint index;
+
+        g_value_init (&new_formats, GST_TYPE_LIST);
+
+        for (index = 0; index < list_size; index++) {
+          const GValue *list_item = gst_value_list_get_value (formats, index);
+
+          info =
+              gst_video_format_get_info (gst_video_format_from_string
+              (g_value_get_string (list_item)));
+          if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) {
+            has_format = TRUE;
+            gst_value_list_append_value (&new_formats, list_item);
+          }
+        }
+
+      } else if (G_VALUE_HOLDS_STRING (formats)) {
+        info =
+            gst_video_format_get_info (gst_video_format_from_string
+            (g_value_get_string (formats)));
+        if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) {
+          has_format = TRUE;
+          gst_value_init_and_copy (&new_formats, formats);
+        }
+
+      } else {
+        g_assert_not_reached ();
+        GST_WARNING ("Unexpected type for video 'format' field: %s",
+            G_VALUE_TYPE_NAME (formats));
+      }
+
+      if (has_format) {
+        s = gst_structure_copy (s);
+        gst_structure_take_value (s, "format", &new_formats);
+        gst_caps_append_structure (result, s);
+      }
+
+    }
+  }
+
+  gst_caps_unref (templatecaps);
+
+  return result;
+}
+
+static GMutex sink_caps_mutex;
+
 static void
-gst_videoaggregator_init (GstVideoAggregator * vagg)
+gst_videoaggregator_init (GstVideoAggregator * vagg,
+    GstVideoAggregatorClass * klass)
 {
+
   vagg->priv =
       G_TYPE_INSTANCE_GET_PRIVATE (vagg, GST_TYPE_VIDEO_AGGREGATOR,
       GstVideoAggregatorPrivate);
@@ -1984,6 +2162,13 @@ gst_videoaggregator_init (GstVideoAggregator * vagg)
   vagg->priv->current_caps = NULL;
 
   g_mutex_init (&vagg->priv->lock);
+
   /* initialize variables */
+  g_mutex_lock (&sink_caps_mutex);
+  if (klass->sink_non_alpha_caps == NULL) {
+    klass->sink_non_alpha_caps = _get_non_alpha_caps_from_template (klass);
+  }
+  g_mutex_unlock (&sink_caps_mutex);
+
   gst_videoaggregator_reset (vagg);
 }
diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h
index b3c5c53e7..f95d0d267 100644
--- a/gst-libs/gst/video/gstvideoaggregator.h
+++ b/gst-libs/gst/video/gstvideoaggregator.h
@@ -111,6 +111,8 @@ struct _GstVideoAggregatorClass
 
   gboolean           preserve_update_caps_result;
 
+  GstCaps           *sink_non_alpha_caps;
+
   /* < private > */
   gpointer            _gst_reserved[GST_PADDING_LARGE];
 };