gsth264parser: Fix handling of NALs with emulation byte set

In case a set of NALs with emulation_prevention_three_byte is decoded using
hardware decoder like Hantro G1, wrong struct v4l2_ctrl_h264_decode_params
.dec_ref_pic_marking_bit_size is passed into the kernel, which results in
decoding artifacts. Subtract the number of emulation three bytes from the
.dec_ref_pic_m->bit_size to get the correct bit size and avoid having these
artifacts. Apply the exact same fix to slice->pic_order_cnt_bit_size as well.

The following NALs (7, 8, 6, 5) decode with artifacts,
.dec_ref_pic_marking_bit_size is set to 10 without this patch.
00000000  00 00 00 01 27 4d 00 20  89 8b 60 3c 04 bf 2e 02  |....'M. ..`<....|
00000010  d4 18 04 18 c0 c0 01 77  00 00 5d c1 7b df 05 00  |.......w..].{...|
00000020  00 00 01 28 ee 1f 20 00  00 01 06 05 10 b9 ed b9  |...(.. .........|
00000030  30 5d 21 4b 71 83 71 2c  10 a3 14 bb 29 80 00 00  |0]!Kq.q,....)...|
00000040  01 25 b8 00 05 00 00 03  03 7f fa 78 1e e7 fd fe  |.%.........x....|
                         ^^^^^^^^^^^^--- emulation 3 byte
00000050  b4 62 7a 31 ff 7d 81 fd  26 d8 62 b6 d6 25 46 ae  |.bz1.}..&.b..%F.|

The following NALs (7, 8, 6, 5) decode fine,
.dec_ref_pic_marking_bit_size is set to 2 without this patch.
00000000  00 00 00 01 27 4d 00 20  89 8b 60 3c 04 bf 2e 02  |....'M. ..`<....|
00000010  d4 18 04 18 c0 c0 01 77  00 00 5d c1 7b df 05 00  |.......w..].{...|
00000020  00 00 01 28 ee 1f 20 00  00 01 06 05 10 b9 ed b9  |...(.. .........|
00000030  30 5d 21 4b 71 83 71 2c  10 a3 14 bb 29 80 00 00  |0]!Kq.q,....)...|
00000040  01 25 b8 00 04 c0 00 03  7f fa 78 1e e7 fd fe b4  |.%........x.....|
00000050  62 7a 31 ff 7d 81 fd 26  d8 62 b6 d6 25 46 ae ce  |bz1.}..&.b..%F..|

Fixes: d0d65fa875 ("codecparsers: h264: record dec_ref_pic_marking() size")
Fixes: 0cc7d6f093 ("codecparsers: h264: record pic_order_cnt elements size")
Signed-off-by: Marek Vasut <marex@denx.de>
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2517>

1 files changed, 8 insertions, 4 deletions

diff --git a/gst-libs/gst/codecparsers/gsth264parser.c b/gst-libs/gst/codecparsers/gsth264parser.c
index 4aaced912..31ebb93c0 100644
--- a/gst-libs/gst/codecparsers/gsth264parser.c
+++ b/gst-libs/gst/codecparsers/gsth264parser.c
@@ -704,11 +704,12 @@ gst_h264_slice_parse_dec_ref_pic_marking (GstH264SliceHdr * slice,
     GstH264NalUnit * nalu, NalReader * nr)
 {
   GstH264DecRefPicMarking *dec_ref_pic_m;
-  guint start_pos;
+  guint start_pos, start_epb;
 
   GST_DEBUG ("parsing \"Decoded reference picture marking\"");
 
   start_pos = nal_reader_get_pos (nr);
+  start_epb = nal_reader_get_epb_count (nr);
 
   dec_ref_pic_m = &slice->dec_ref_pic_marking;
 
@@ -753,7 +754,8 @@ gst_h264_slice_parse_dec_ref_pic_marking (GstH264SliceHdr * slice,
     }
   }
 
-  dec_ref_pic_m->bit_size = nal_reader_get_pos (nr) - start_pos;
+  dec_ref_pic_m->bit_size = (nal_reader_get_pos (nr) - start_pos) -
+      (8 * (nal_reader_get_epb_count (nr) - start_epb));
 
   return TRUE;
 
@@ -2228,7 +2230,7 @@ gst_h264_parser_parse_slice_hdr (GstH264NalParser * nalparser,
   gint pps_id;
   GstH264PPS *pps;
   GstH264SPS *sps;
-  guint start_pos;
+  guint start_pos, start_epb;
 
   memset (slice, 0, sizeof (*slice));
 
@@ -2304,6 +2306,7 @@ gst_h264_parser_parse_slice_hdr (GstH264NalParser * nalparser,
     READ_UE_MAX (&nr, slice->idr_pic_id, G_MAXUINT16);
 
   start_pos = nal_reader_get_pos (&nr);
+  start_epb = nal_reader_get_epb_count (&nr);
 
   if (sps->pic_order_cnt_type == 0) {
     READ_UINT16 (&nr, slice->pic_order_cnt_lsb,
@@ -2319,7 +2322,8 @@ gst_h264_parser_parse_slice_hdr (GstH264NalParser * nalparser,
       READ_SE (&nr, slice->delta_pic_order_cnt[1]);
   }
 
-  slice->pic_order_cnt_bit_size = nal_reader_get_pos (&nr) - start_pos;
+  slice->pic_order_cnt_bit_size = (nal_reader_get_pos (&nr) - start_pos) -
+      (8 * (nal_reader_get_epb_count (&nr) - start_epb));
 
   if (pps->redundant_pic_cnt_present_flag)
     READ_UE_MAX (&nr, slice->redundant_pic_cnt, G_MAXINT8);