vtdec: Prefer GL output caps.

Prefer GL caps by putting them first in the pad template and
intersecting using GST_CAPS_INTERSECT_FIRST.

1 files changed, 33 insertions, 23 deletions

diff --git a/sys/applemedia/vtdec.c b/sys/applemedia/vtdec.c
index 6a6240b79..1cf902cee 100644
--- a/sys/applemedia/vtdec.c
+++ b/sys/applemedia/vtdec.c
@@ -109,10 +109,10 @@ CFSTR ("RequireHardwareAcceleratedVideoDecoder");
 #endif
 
 #define VIDEO_SRC_CAPS \
-    GST_VIDEO_CAPS_MAKE(GST_VTDEC_VIDEO_FORMAT_STR) ";" \
     GST_VIDEO_CAPS_MAKE_WITH_FEATURES \
     (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, \
-        "RGBA") ";"
+        "RGBA") ";" \
+    GST_VIDEO_CAPS_MAKE(GST_VTDEC_VIDEO_FORMAT_STR) ";"
 
 G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER);
 
@@ -244,24 +244,45 @@ out:
   return ret;
 }
 
-static GstVideoFormat
-gst_vtdec_negotiate_output_format (GstVtdec * vtdec)
+static gboolean
+gst_vtdec_negotiate_output_format (GstVtdec * vtdec, GstVideoCodecState *input_state)
 {
-  GstCaps *caps = NULL;
-  GstVideoFormat format;
+  GstCaps *caps = NULL, *peercaps = NULL, *templcaps;
+  GstVideoFormat output_format;
+  GstVideoCodecState *output_state = NULL;
+  GstCapsFeatures *features;
   GstStructure *structure;
   const gchar *s;
 
-  caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec));
-  if (!caps)
-    caps = gst_pad_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL);
+  peercaps = gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL);
+
+  /* Check if output supports GL caps by preference */
+  templcaps = gst_pad_get_pad_template_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec));
+  caps = gst_caps_intersect_full (templcaps, peercaps, GST_CAPS_INTERSECT_FIRST);
+
+  gst_caps_unref (peercaps);
+  gst_caps_unref (templcaps);
+
   caps = gst_caps_truncate (caps);
   structure = gst_caps_get_structure (caps, 0);
   s = gst_structure_get_string (structure, "format");
-  format = gst_video_format_from_string (s);
+  output_format = gst_video_format_from_string (s);
+  features = gst_caps_features_copy (gst_caps_get_features (caps, 0));
+
   gst_caps_unref (caps);
 
-  return format;
+  if (!gst_vtdec_create_session (vtdec, output_format)) {
+    gst_caps_features_free (features);
+    return FALSE;
+  }
+
+  output_state = gst_video_decoder_set_output_state (GST_VIDEO_DECODER (vtdec),
+      output_format, vtdec->video_info.width, vtdec->video_info.height, input_state);
+
+  output_state->caps = gst_video_info_to_caps (&output_state->info);
+  gst_caps_set_features (output_state->caps, 0, features);
+
+  return TRUE;
 }
 
 static gboolean
@@ -272,8 +293,6 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
   CMFormatDescriptionRef format_description = NULL;
   const char *caps_name;
   GstVtdec *vtdec = GST_VTDEC (decoder);
-  GstVideoFormat output_format;
-  GstVideoCodecState *output_state;
 
   GST_DEBUG_OBJECT (vtdec, "set_format");
 
@@ -313,18 +332,9 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
     CFRelease (vtdec->format_description);
   vtdec->format_description = format_description;
 
-  output_format = gst_vtdec_negotiate_output_format (vtdec);
-  if (!gst_vtdec_create_session (vtdec, output_format))
+  if (!gst_vtdec_negotiate_output_format (vtdec, state))
     return FALSE;
 
-  output_state = gst_video_decoder_set_output_state (decoder,
-      output_format, vtdec->video_info.width, vtdec->video_info.height, state);
-  output_state->caps = gst_video_info_to_caps (&output_state->info);
-  if (output_state->info.finfo->format == GST_VIDEO_FORMAT_RGBA) {
-    gst_caps_set_features (output_state->caps, 0,
-        gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL));
-  }
-
   return TRUE;
 }