v4l2: videodec: Prefer acquired caps over anything downstream

As we don't have anything smart in the fixation process, we may endup with
a format that has a lower bitdepth, even if downstream can handle higher
depth. it is notably the case when negotiating with deinterlace, which places
is non-passthrough caps before its passthrough one. This makes the generic
fixation prefer the formats natively supported by deinterlace element over
the HW 10bit format. As some HW can downscale 10bit to 8bit, this can break
10bit decoding.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4426>

2 files changed, 21 insertions, 12 deletions

diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2bufferpool.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2bufferpool.c
index 5f564e5688..889447f79d 100644
--- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2bufferpool.c
+++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2bufferpool.c
@@ -2333,8 +2333,6 @@ gst_v4l2_buffer_pool_enable_resolution_change (GstV4l2BufferPool * pool)
 {
   guint32 input_id = 0;
 
-  g_return_if_fail (!gst_buffer_pool_is_active (GST_BUFFER_POOL (pool)));
-
   /* Make sure we subscribe for the current input */
   gst_v4l2_get_input (pool->obj, &input_id);
 
diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c
index 7a32783a20..8550910716 100644
--- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c
+++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2videodec.c
@@ -570,7 +570,7 @@ gst_v4l2_video_dec_setup_capture (GstVideoDecoder * decoder)
   GstV4l2Error error = GST_V4L2_ERROR_INIT;
   GstVideoInfo info;
   GstVideoCodecState *output_state;
-  GstCaps *acquired_caps, *available_caps, *caps, *filter;
+  GstCaps *acquired_caps, *fixation_caps, *available_caps, *caps, *filter;
   GstStructure *st;
   GstBufferPool *cpool;
   gboolean active;
@@ -598,7 +598,8 @@ gst_v4l2_video_dec_setup_capture (GstVideoDecoder * decoder)
     /* Create caps from the acquired format, remove the format field */
     acquired_caps = gst_video_info_to_caps (&info);
     GST_DEBUG_OBJECT (self, "Acquired caps: %" GST_PTR_FORMAT, acquired_caps);
-    st = gst_caps_get_structure (acquired_caps, 0);
+    fixation_caps = gst_caps_copy (acquired_caps);
+    st = gst_caps_get_structure (fixation_caps, 0);
     gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site",
         NULL);
 
@@ -610,10 +611,10 @@ gst_v4l2_video_dec_setup_capture (GstVideoDecoder * decoder)
      * with downstream, not coded size. */
     gst_caps_map_in_place (available_caps, gst_v4l2_video_remove_padding, self);
 
-    filter = gst_caps_intersect_full (available_caps, acquired_caps,
+    filter = gst_caps_intersect_full (available_caps, fixation_caps,
         GST_CAPS_INTERSECT_FIRST);
     GST_DEBUG_OBJECT (self, "Filtered caps: %" GST_PTR_FORMAT, filter);
-    gst_caps_unref (acquired_caps);
+    gst_caps_unref (fixation_caps);
     gst_caps_unref (available_caps);
     caps = gst_pad_peer_query_caps (decoder->srcpad, filter);
     gst_caps_unref (filter);
@@ -624,6 +625,14 @@ gst_v4l2_video_dec_setup_capture (GstVideoDecoder * decoder)
       goto not_negotiated;
     }
 
+    /* Prefer the acquired caps over anything suggested downstream, this ensure
+     * that we preserves the bit depth, as we don't have any fancy fixation
+     * process */
+    if (gst_caps_is_subset (acquired_caps, caps)) {
+      gst_caps_unref (acquired_caps);
+      goto use_acquired_caps;
+    }
+
     /* Fixate pixel format */
     caps = gst_caps_fixate (caps);
 
@@ -634,6 +643,8 @@ gst_v4l2_video_dec_setup_capture (GstVideoDecoder * decoder)
       gst_video_info_from_caps (&info, caps);
     else
       gst_v4l2_clear_error (&error);
+
+  use_acquired_caps:
     gst_caps_unref (caps);
 
     output_state = gst_video_decoder_set_output_state (decoder,
@@ -644,19 +655,19 @@ gst_v4l2_video_dec_setup_capture (GstVideoDecoder * decoder)
     output_state->info.colorimetry = info.colorimetry;
     gst_video_codec_state_unref (output_state);
 
-    cpool = gst_v4l2_object_get_buffer_pool (self->v4l2capture);
-    gst_v4l2_buffer_pool_enable_resolution_change (GST_V4L2_BUFFER_POOL
-        (cpool));
-
     if (!gst_video_decoder_negotiate (decoder)) {
-      if (cpool)
-        gst_object_unref (cpool);
       if (GST_PAD_IS_FLUSHING (decoder->srcpad))
         goto flushing;
       else
         goto not_negotiated;
     }
 
+    /* The pool may be created through gst_video_decoder_negotiate(), so must
+     * be kept after */
+    cpool = gst_v4l2_object_get_buffer_pool (self->v4l2capture);
+    gst_v4l2_buffer_pool_enable_resolution_change (GST_V4L2_BUFFER_POOL
+        (cpool));
+
     /* Ensure our internal pool is activated */
     active = gst_buffer_pool_set_active (cpool, TRUE);
     if (cpool)