rtpvp*depay: possibly forward might-have-been-fec PacketLost events

This is ad adaptation of a Pexip patch for dealing with spurious
GstRTPPacketLost events caused by lost ulpfec packets: as FEC packets
under that scheme are spliced in the same sequence domain as the media
packets, it is not generally possible to determine whether a lost packet
was a FEC packet or a media packet.

When upstreaming pexip's ulpfec patches, we decided to drop all lost
events at the base depayloader level, and where the original patch
from pexip was making use of picture ids and marker bits to determine
whether a packet should be forwarded, this patch makes use of those
to determine whether they should be dropped instead (by removing their
might-have-been-fec field).

Spurious lost events coming out of the depayloader can cause the
decoder to stop decoding until the next keyframe and / or request a new
keyframe, and while this is not desirable it makes sense to forward
that information when we have other means to determine whether a lost
packet was indeed a FEC packet, as is the case with VP8 / VP9 payloads
when they carry a picture id.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/769>

2 files changed, 405 insertions, 2 deletions

diff --git a/tests/check/elements/rtpvp8.c b/tests/check/elements/rtpvp8.c
index 666bed5ed..44112898a 100644
--- a/tests/check/elements/rtpvp8.c
+++ b/tests/check/elements/rtpvp8.c
@@ -29,6 +29,56 @@
       g_memdup (vp8_bitstream_payload, sizeof (vp8_bitstream_payload)), \
       sizeof (vp8_bitstream_payload))
 
+static guint8 intra_picid6336_seqnum0[] = {
+  0x80, 0xe0, 0x00, 0x00, 0x9a, 0xbb, 0xe3, 0xb3, 0x8b, 0xe9, 0x1d, 0x61,
+  0x90, 0x80, 0x98, 0xc0, 0xf0, 0x07, 0x00, 0x9d, 0x01, 0x2a, 0xb0, 0x00,
+  0x90, 0x00, 0x06, 0x47, 0x08, 0x85, 0x85, 0x88, 0x99, 0x84, 0x88, 0x21,
+};
+
+static guint8 intra_picid24_seqnum0[] = {
+  0x80, 0xe0, 0x00, 0x00, 0x9a, 0xbb, 0xe3, 0xb3, 0x8b, 0xe9, 0x1d, 0x61,
+  0x90, 0x80, 0x18, 0xf0, 0x07, 0x00, 0x9d, 0x01, 0x2a, 0xb0, 0x00,
+  0x90, 0x00, 0x06, 0x47, 0x08, 0x85, 0x85, 0x88, 0x99, 0x84, 0x88, 0x21,
+};
+
+static guint8 intra_nopicid_seqnum0[] = {
+  0x80, 0xe0, 0x00, 0x00, 0x9a, 0xbb, 0xe3, 0xb3, 0x8b, 0xe9, 0x1d, 0x61,
+  0x90, 0x00, 0xf0, 0x07, 0x00, 0x9d, 0x01, 0x2a, 0xb0, 0x00,
+  0x90, 0x00, 0x06, 0x47, 0x08, 0x85, 0x85, 0x88, 0x99, 0x84, 0x88, 0x21,
+};
+
+static GstBuffer *
+create_rtp_vp8_buffer (guint seqnum, guint picid, gint picid_bits,
+    GstClockTime buf_pts)
+{
+  GstBuffer *ret;
+  guint8 *packet;
+  gsize size;
+
+  g_assert (picid_bits == 0 || picid_bits == 7 || picid_bits == 15);
+
+  if (picid_bits == 0) {
+    size = sizeof (intra_nopicid_seqnum0);
+    packet = g_memdup (intra_nopicid_seqnum0, size);
+  } else if (picid_bits == 7) {
+    size = sizeof (intra_picid24_seqnum0);
+    packet = g_memdup (intra_picid24_seqnum0, size);
+    packet[14] = picid & 0x7f;
+  } else {
+    size = sizeof (intra_picid6336_seqnum0);
+    packet = g_memdup (intra_picid6336_seqnum0, size);
+    packet[14] = ((picid >> 8) & 0xff) | 0x80;
+    packet[15] = (picid >> 0) & 0xff;
+  }
+
+  packet[2] = (seqnum >> 8) & 0xff;
+  packet[3] = (seqnum >> 0) & 0xff;
+
+  ret = gst_buffer_new_wrapped (packet, size);
+  GST_BUFFER_PTS (ret) = buf_pts;
+  return ret;
+}
+
 static void
 add_vp8_meta (GstBuffer * buffer, gboolean use_temporal_scaling,
     gboolean layer_sync, guint layer_id, guint tl0picidx)
@@ -445,6 +495,107 @@ GST_START_TEST (test_pay_tl0picidx_split_buffer)
 
 GST_END_TEST;
 
+typedef struct _DepayGapEventTestData
+{
+  gint seq_num;
+  gint picid;
+  gint picid_bits;
+} DepayGapEventTestData;
+
+static void
+test_depay_gap_event_base (const DepayGapEventTestData * data,
+    gboolean send_lost_event, gboolean expect_gap_event)
+{
+  GstEvent *event;
+  GstClockTime pts = 0;
+  GstHarness *h = gst_harness_new ("rtpvp8depay");
+  gst_harness_set_src_caps_str (h, RTP_VP8_CAPS_STR);
+
+  gst_harness_push (h, create_rtp_vp8_buffer (data[0].seq_num, data[0].picid,
+          data[0].picid_bits, pts));
+  pts += 33 * GST_MSECOND;
+
+  /* Preparation before pushing gap event. Getting rid of all events which
+   * came by this point - segment, caps, etc */
+  for (gint i = 0; i < 3; i++)
+    gst_event_unref (gst_harness_pull_event (h));
+  fail_unless_equals_int (gst_harness_events_in_queue (h), 0);
+
+  if (send_lost_event) {
+    gst_harness_push_event (h,
+        gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM,
+            gst_structure_new ("GstRTPPacketLost", "timestamp", G_TYPE_UINT64,
+                pts, "duration", G_TYPE_UINT64, 33 * GST_MSECOND,
+                "might-have-been-fec", G_TYPE_BOOLEAN, TRUE, NULL)));
+    pts += 33 * GST_MSECOND;
+  }
+
+  gst_harness_push (h, create_rtp_vp8_buffer (data[1].seq_num, data[1].picid,
+          data[1].picid_bits, pts));
+  fail_unless_equals_int (2, gst_harness_buffers_received (h));
+
+  if (expect_gap_event) {
+    /* Making sure the GAP event was pushed downstream */
+    event = gst_harness_pull_event (h);
+    fail_unless_equals_string ("gap",
+        gst_event_type_get_name (GST_EVENT_TYPE (event)));
+    gst_event_unref (event);
+  }
+  fail_unless_equals_int (gst_harness_events_in_queue (h), 0);
+
+  gst_harness_teardown (h);
+}
+
+/* Packet loss + no loss in picture ids */
+static const DepayGapEventTestData stop_gap_events_test_data[][2] = {
+  /* 7bit picture ids */
+  {{100, 24, 7}, {102, 25, 7}},
+
+  /* 15bit picture ids */
+  {{100, 250, 15}, {102, 251, 15}},
+
+  /* 7bit picture ids wrap */
+  {{100, 127, 7}, {102, 0, 7}},
+
+  /* 15bit picture ids wrap */
+  {{100, 32767, 15}, {102, 0, 15}},
+
+  /* 7bit to 15bit picture id */
+  {{100, 127, 7}, {102, 128, 15}},
+};
+
+GST_START_TEST (test_depay_stop_gap_events)
+{
+  test_depay_gap_event_base (&stop_gap_events_test_data[__i__][0], TRUE, FALSE);
+}
+
+GST_END_TEST;
+
+/* Packet loss + lost picture ids */
+static const DepayGapEventTestData resend_gap_event_test_data[][2] = {
+  /* 7bit picture ids */
+  {{100, 24, 7}, {102, 26, 7}},
+
+  /* 15bit picture ids */
+  {{100, 250, 15}, {102, 252, 15}},
+
+  /* 7bit picture ids wrap */
+  {{100, 127, 7}, {102, 1, 7}},
+
+  /* 15bit picture ids wrap */
+  {{100, 32767, 15}, {102, 1, 15}},
+
+  /* 7bit to 15bit picture id */
+  {{100, 126, 7}, {102, 129, 15}},
+};
+
+GST_START_TEST (test_depay_resend_gap_event)
+{
+  test_depay_gap_event_base (&resend_gap_event_test_data[__i__][0], TRUE, TRUE);
+}
+
+GST_END_TEST;
+
 static Suite *
 rtpvp8_suite (void)
 {
@@ -462,6 +613,10 @@ rtpvp8_suite (void)
       G_N_ELEMENTS (with_meta_test_data));
   tcase_add_test (tc_chain, test_pay_continuous_picture_id_and_tl0picidx);
   tcase_add_test (tc_chain, test_pay_tl0picidx_split_buffer);
+  tcase_add_loop_test (tc_chain, test_depay_stop_gap_events, 0,
+      G_N_ELEMENTS (stop_gap_events_test_data));
+  tcase_add_loop_test (tc_chain, test_depay_resend_gap_event, 0,
+      G_N_ELEMENTS (resend_gap_event_test_data));
 
   return s;
 }
diff --git a/tests/check/elements/rtpvp9.c b/tests/check/elements/rtpvp9.c
index 123069e86..f50ce3ba0 100644
--- a/tests/check/elements/rtpvp9.c
+++ b/tests/check/elements/rtpvp9.c
@@ -25,7 +25,6 @@
 #define RTP_VP9_CAPS_STR \
   "application/x-rtp,media=video,encoding-name=VP9,clock-rate=90000,payload=96"
 
-
 GST_START_TEST (test_depay_flexible_mode)
 {
   /* b-bit, e-bit, f-bit and marker bit set */
@@ -96,17 +95,266 @@ GST_START_TEST (test_depay_non_flexible_mode)
 
 GST_END_TEST;
 
+static guint8 intra_picid6336_seqnum0[] = {
+  0x80, 0xf4, 0x00, 0x00, 0x49, 0x88, 0xd9, 0xf8, 0xa0, 0x6c, 0x65, 0x6c,
+  0x8c, 0x98, 0xc0, 0x87, 0x01, 0x02, 0x49, 0x3f, 0x1c, 0x12, 0x0e, 0x0c,
+  0xd0, 0x1b, 0xb9, 0x80, 0x80, 0xb0, 0x18, 0x0f, 0xa6, 0x4d, 0x01, 0xa5
+};
+
+static guint8 intra_picid24_seqnum0[] = {
+  0x80, 0xf4, 0x00, 0x00, 0x49, 0x88, 0xd9, 0xf8, 0xa0, 0x6c, 0x65, 0x6c,
+  0x8c, 0x18, 0x87, 0x01, 0x02, 0x49, 0x3f, 0x1c, 0x12, 0x0e, 0x0c,
+  0xd0, 0x1b, 0xb9, 0x80, 0x80, 0xb0, 0x18, 0x0f, 0xa6, 0x4d, 0x01, 0xa5
+};
+
+static guint8 intra_nopicid_seqnum0[] = {
+  0x80, 0xf4, 0x00, 0x00, 0x49, 0x88, 0xd9, 0xf8, 0xa0, 0x6c, 0x65, 0x6c,
+  0x0c, 0x87, 0x01, 0x02, 0x49, 0x3f, 0x1c, 0x12, 0x0e, 0x0c,
+  0xd0, 0x1b, 0xb9, 0x80, 0x80, 0xb0, 0x18, 0x0f, 0xa6, 0x4d, 0x01, 0xa5
+};
+
+enum
+{
+  BT_PLAIN_PICID_NONE,
+  BT_PLAIN_PICID_7,
+  BT_PLAIN_PICID_15,
+  /* Commented out for now, until added VP9 equvivalents.
+     BT_TS_PICID_NONE,
+     BT_TS_PICID_7,
+     BT_TS_PICID_15,
+     BT_TS_PICID_7_NO_TLOPICIDX,
+     BT_TS_PICID_7_NO_TID_Y_KEYIDX
+   */
+};
+
+static GstBuffer *
+create_rtp_vp9_buffer_full (guint seqnum, guint picid, guint buffer_type,
+    GstClockTime buf_pts, gboolean B_bit_start_of_frame, gboolean marker_bit)
+{
+  static struct BufferTemplate
+  {
+    guint8 *template;
+    gsize size;
+    gint picid_bits;
+  } templates[] = {
+    {
+    intra_nopicid_seqnum0, sizeof (intra_nopicid_seqnum0), 0}
+    , {
+    intra_picid24_seqnum0, sizeof (intra_picid24_seqnum0), 7}
+    , {
+    intra_picid6336_seqnum0, sizeof (intra_picid6336_seqnum0), 15}
+    ,
+        /*
+           { intra_nopicid_seqnum0_tl1_sync_tl0picidx12,
+           sizeof (intra_nopicid_seqnum0_tl1_sync_tl0picidx12),
+           0
+           },
+           { intra_picid24_seqnum0_tl1_sync_tl0picidx12,
+           sizeof (intra_picid24_seqnum0_tl1_sync_tl0picidx12),
+           7
+           },
+           { intra_picid6336_seqnum0_tl1_sync_tl0picidx12,
+           sizeof (intra_picid6336_seqnum0_tl1_sync_tl0picidx12),
+           15
+           },
+           { intra_picid24_seqnum0_tl1_sync_no_tl0picidx,
+           sizeof (intra_picid24_seqnum0_tl1_sync_no_tl0picidx),
+           7
+           },
+           { intra_picid24_seqnum0_notyk_tl0picidx12,
+           sizeof (intra_picid24_seqnum0_notyk_tl0picidx12),
+           7
+           }
+         */
+  };
+  struct BufferTemplate *template = &templates[buffer_type];
+  guint8 *packet = g_memdup (template->template, template->size);
+  GstBuffer *ret;
+
+  packet[2] = (seqnum >> 8) & 0xff;
+  packet[3] = (seqnum >> 0) & 0xff;
+
+  /* We're forcing the E-bit (EndOfFrame) together with the RTP marker bit here, which is a bit of a hack.
+   * If we're to enable spatial scalability tests, we need to take that into account when setting the E bit.
+   */
+  if (marker_bit) {
+    packet[1] |= 0x80;
+    packet[12] |= 0x4;
+  } else {
+    packet[1] &= ~0x80;
+    packet[12] &= ~0x4;
+  }
+
+  if (B_bit_start_of_frame)
+    packet[12] |= 0x8;
+  else
+    packet[12] &= ~0x8;
+
+  if (template->picid_bits == 7) {
+    /* Prerequisites for this to be correct:
+       ((packet[12] & 0x80) == 0x80); I bit set
+     */
+    g_assert ((packet[12] & 0x80) == 0x80);
+    packet[13] = picid & 0x7f;
+
+  } else if (template->picid_bits == 15) {
+    /* Prerequisites for this to be correct:
+       ((packet[12] & 0x80) == 0x80); I bit set
+     */
+    g_assert ((packet[12] & 0x80) == 0x80);
+    packet[13] = ((picid >> 8) & 0xff) | 0x80;
+    packet[14] = (picid >> 0) & 0xff;
+  }
+
+  ret = gst_buffer_new_wrapped (packet, template->size);
+  GST_BUFFER_PTS (ret) = buf_pts;
+  return ret;
+}
+
+static GstBuffer *
+create_rtp_vp9_buffer (guint seqnum, guint picid, guint buffer_type,
+    GstClockTime buf_pts)
+{
+  return create_rtp_vp9_buffer_full (seqnum, picid, buffer_type, buf_pts, TRUE,
+      TRUE);
+}
+
+typedef struct _DepayGapEventTestData
+{
+  gint seq_num;
+  gint picid;
+  guint buffer_type;
+} DepayGapEventTestData;
+
+typedef struct
+{
+  gint seq_num;
+  gint picid;
+  guint buffer_type;
+  gboolean s_bit;
+  gboolean marker_bit;
+} DepayGapEventTestDataFull;
+
+static void
+test_depay_gap_event_base (const DepayGapEventTestData * data,
+    gboolean send_lost_event, gboolean expect_gap_event, int iter)
+{
+  GstEvent *event;
+  GstClockTime pts = 0;
+  GstHarness *h = gst_harness_new ("rtpvp9depay");
+  if (send_lost_event == FALSE && expect_gap_event) {
+    /* Expect picture ID gaps to be concealed, so tell the element to do so. */
+    g_object_set (h->element, "hide-picture-id-gap", TRUE, NULL);
+  }
+  gst_harness_set_src_caps_str (h, RTP_VP9_CAPS_STR);
+
+  gst_harness_push (h, create_rtp_vp9_buffer (data[0].seq_num, data[0].picid,
+          data[0].buffer_type, pts));
+  pts += 33 * GST_MSECOND;
+
+  /* Preparation before pushing gap event. Getting rid of all events which
+   * came by this point - segment, caps, etc */
+  for (gint i = 0; i < 3; i++)
+    gst_event_unref (gst_harness_pull_event (h));
+  fail_unless_equals_int (gst_harness_events_in_queue (h), 0);
+
+  if (send_lost_event) {
+    gst_harness_push_event (h,
+        gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM,
+            gst_structure_new ("GstRTPPacketLost", "timestamp", G_TYPE_UINT64,
+                pts, "duration", G_TYPE_UINT64, 33 * GST_MSECOND,
+                "might-have-been-fec", G_TYPE_BOOLEAN, TRUE, NULL)));
+    pts += 33 * GST_MSECOND;
+  }
+
+  gst_harness_push (h, create_rtp_vp9_buffer (data[1].seq_num, data[1].picid,
+          data[1].buffer_type, pts));
+  fail_unless_equals_int (2, gst_harness_buffers_received (h));
+
+  if (expect_gap_event) {
+    gboolean noloss = FALSE;
+
+    /* Making sure the GAP event was pushed downstream */
+    event = gst_harness_pull_event (h);
+    fail_unless_equals_string ("gap",
+        gst_event_type_get_name (GST_EVENT_TYPE (event)));
+    gst_structure_get_boolean (gst_event_get_structure (event),
+        "no-packet-loss", &noloss);
+
+    /* If we didn't send GstRTPPacketLost event, the gap
+     * event should indicate that with 'no-packet-loss' parameter */
+    fail_unless_equals_int (noloss, !send_lost_event);
+    gst_event_unref (event);
+  }
+
+  fail_unless_equals_int (gst_harness_events_in_queue (h), 0);
+
+  gst_harness_teardown (h);
+}
+
+static const DepayGapEventTestData stop_gap_events_test_data[][2] = {
+  /* 7bit picture ids */
+  {{100, 24, BT_PLAIN_PICID_7}, {102, 25, BT_PLAIN_PICID_7}},
+
+  /* 15bit picture ids */
+  {{100, 250, BT_PLAIN_PICID_15}, {102, 251, BT_PLAIN_PICID_15}},
+
+  /* 7bit picture ids wrap */
+  {{100, 127, BT_PLAIN_PICID_7}, {102, 0, BT_PLAIN_PICID_7}},
+
+  /* 15bit picture ids wrap */
+  {{100, 32767, BT_PLAIN_PICID_15}, {102, 0, BT_PLAIN_PICID_15}},
+
+  /* 7bit to 15bit picture id */
+  {{100, 127, BT_PLAIN_PICID_7}, {102, 128, BT_PLAIN_PICID_15}},
+};
+
+GST_START_TEST (test_depay_stop_gap_events)
+{
+  test_depay_gap_event_base (&stop_gap_events_test_data[__i__][0], TRUE, FALSE,
+      __i__);
+}
+
+GST_END_TEST;
+
+/* Packet loss + lost picture ids */
+static const DepayGapEventTestData resend_gap_event_test_data[][2] = {
+  /* 7bit picture ids */
+  {{100, 24, BT_PLAIN_PICID_7}, {102, 26, BT_PLAIN_PICID_7}},
+
+  /* 15bit picture ids */
+  {{100, 250, BT_PLAIN_PICID_15}, {102, 252, BT_PLAIN_PICID_15}},
+
+  /* 7bit picture ids wrap */
+  {{100, 127, BT_PLAIN_PICID_7}, {102, 1, BT_PLAIN_PICID_7}},
 
+  /* 15bit picture ids wrap */
+  {{100, 32767, BT_PLAIN_PICID_15}, {102, 1, BT_PLAIN_PICID_15}},
+
+  /* 7bit to 15bit picture id */
+  {{100, 126, BT_PLAIN_PICID_7}, {102, 129, BT_PLAIN_PICID_15}},
+};
+
+GST_START_TEST (test_depay_resend_gap_event)
+{
+  test_depay_gap_event_base (&resend_gap_event_test_data[__i__][0], TRUE, TRUE,
+      __i__);
+}
+
+GST_END_TEST;
 
 static Suite *
 rtpvp9_suite (void)
 {
   Suite *s = suite_create ("rtpvp9");
   TCase *tc_chain;
-
   suite_add_tcase (s, (tc_chain = tcase_create ("vp9depay")));
   tcase_add_test (tc_chain, test_depay_flexible_mode);
   tcase_add_test (tc_chain, test_depay_non_flexible_mode);
+  tcase_add_loop_test (tc_chain, test_depay_stop_gap_events, 0,
+      G_N_ELEMENTS (stop_gap_events_test_data));
+  tcase_add_loop_test (tc_chain, test_depay_resend_gap_event, 0,
+      G_N_ELEMENTS (resend_gap_event_test_data));
 
   return s;
 }