cccombiner: implement scheduling

Prior to that, cccombiner's behaviour was essentially that of
a funnel: it strictly looked at input timestamps to associate
together video and caption buffers.

This patch instead exposes a "schedule" property, with a default
of TRUE, to control whether caption buffers should be smoothly
scheduled, in order to have exactly one per output video buffer.

This can involve rewriting input captions, for example when the
input is CDP sequence counters are rewritten, time codes are dropped
and potentially re-injected if the input video frame had a time code
meta.

Caption buffers may also get split up in order to assign captions to
the correct field when the input is interlaced.

This can also imply that the input will drift from synchronization,
when there isn't enough padding in the input stream to catch up. In
that case the element will start dropping old caption buffers once
the number of buffers in its internal queue reaches a certain limit
(configurable).

The property is exposed so that existing users of cccombiner can
revert back to the original behaviour, but should eventually be
removed, as that behaviour was simply inadequate.

This commit also disallows changing the input caption type, as
this would needlessly complicate implementation, and removes
the corresponding test.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2076>

4 files changed, 919 insertions, 114 deletions

diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json
index 9f995d555..6eae76468 100644
--- a/docs/plugins/gst_plugins_cache.json
+++ b/docs/plugins/gst_plugins_cache.json
@@ -3318,7 +3318,34 @@
                         "type": "GstAggregatorPad"
                     }
                 },
-                "properties": {},
+                "properties": {
+                    "max-scheduled": {
+                        "blurb": "Maximum number of buffers to queue for scheduling",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "0",
+                        "max": "-1",
+                        "min": "0",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "guint",
+                        "writable": true
+                    },
+                    "schedule": {
+                        "blurb": "Schedule caption buffers so that exactly one is output per video frame",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "true",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "gboolean",
+                        "writable": true
+                    }
+                },
                 "rank": "none"
             },
             "ccconverter": {
diff --git a/ext/closedcaption/gstcccombiner.c b/ext/closedcaption/gstcccombiner.c
index f1781c6cb..5e47318f5 100644
--- a/ext/closedcaption/gstcccombiner.c
+++ b/ext/closedcaption/gstcccombiner.c
@@ -54,12 +54,29 @@ static GstStaticPadTemplate captiontemplate =
 G_DEFINE_TYPE (GstCCCombiner, gst_cc_combiner, GST_TYPE_AGGREGATOR);
 #define parent_class gst_cc_combiner_parent_class
 
+enum
+{
+  PROP_0,
+  PROP_SCHEDULE,
+  PROP_MAX_SCHEDULED,
+};
+
+#define DEFAULT_MAX_SCHEDULED 30
+#define DEFAULT_SCHEDULE TRUE
+
 typedef struct
 {
   GstVideoCaptionType caption_type;
   GstBuffer *buffer;
 } CaptionData;
 
+typedef struct
+{
+  GstBuffer *buffer;
+  GstClockTime running_time;
+  GstClockTime stream_time;
+} CaptionQueueItem;
+
 static void
 caption_data_clear (CaptionData * data)
 {
@@ -67,10 +84,18 @@ caption_data_clear (CaptionData * data)
 }
 
 static void
+clear_scheduled (CaptionQueueItem * item)
+{
+  gst_buffer_unref (item->buffer);
+}
+
+static void
 gst_cc_combiner_finalize (GObject * object)
 {
   GstCCCombiner *self = GST_CCCOMBINER (object);
 
+  gst_queue_array_free (self->scheduled[0]);
+  gst_queue_array_free (self->scheduled[1]);
   g_array_unref (self->current_frame_captions);
   self->current_frame_captions = NULL;
 
@@ -79,6 +104,599 @@ gst_cc_combiner_finalize (GObject * object)
 
 #define GST_FLOW_NEED_DATA GST_FLOW_CUSTOM_SUCCESS
 
+static const guint8 *
+extract_cdp (const guint8 * cdp, guint cdp_len, guint * cc_data_len)
+{
+  GstByteReader br;
+  guint16 u16;
+  guint8 u8;
+  guint8 flags;
+  guint len = 0;
+  const guint8 *cc_data = NULL;
+
+  *cc_data_len = 0;
+
+  /* Header + footer length */
+  if (cdp_len < 11) {
+    goto done;
+  }
+
+  gst_byte_reader_init (&br, cdp, cdp_len);
+  u16 = gst_byte_reader_get_uint16_be_unchecked (&br);
+  if (u16 != 0x9669) {
+    goto done;
+  }
+
+  u8 = gst_byte_reader_get_uint8_unchecked (&br);
+  if (u8 != cdp_len) {
+    goto done;
+  }
+
+  gst_byte_reader_skip_unchecked (&br, 1);
+
+  flags = gst_byte_reader_get_uint8_unchecked (&br);
+
+  /* No cc_data? */
+  if ((flags & 0x40) == 0) {
+    goto done;
+  }
+
+  /* cdp_hdr_sequence_cntr */
+  gst_byte_reader_skip_unchecked (&br, 2);
+
+  /* time_code_present */
+  if (flags & 0x80) {
+    if (gst_byte_reader_get_remaining (&br) < 5) {
+      goto done;
+    }
+    gst_byte_reader_skip_unchecked (&br, 5);
+  }
+
+  /* ccdata_present */
+  if (flags & 0x40) {
+    guint8 cc_count;
+
+    if (gst_byte_reader_get_remaining (&br) < 2) {
+      goto done;
+    }
+    u8 = gst_byte_reader_get_uint8_unchecked (&br);
+    if (u8 != 0x72) {
+      goto done;
+    }
+
+    cc_count = gst_byte_reader_get_uint8_unchecked (&br);
+    if ((cc_count & 0xe0) != 0xe0) {
+      goto done;
+    }
+    cc_count &= 0x1f;
+
+    if (cc_count == 0)
+      return 0;
+
+    len = 3 * cc_count;
+    if (gst_byte_reader_get_remaining (&br) < len)
+      goto done;
+
+    cc_data = gst_byte_reader_get_data_unchecked (&br, len);
+    *cc_data_len = len;
+  }
+
+done:
+  return cc_data;
+}
+
+#define MAX_CDP_PACKET_LEN 256
+#define MAX_CEA608_LEN 32
+
+static const struct cdp_fps_entry cdp_fps_table[] = {
+  {0x1f, 24000, 1001, 25, 22, 3 /* FIXME: alternating max cea608 count! */ },
+  {0x2f, 24, 1, 25, 22, 2},
+  {0x3f, 25, 1, 24, 22, 2},
+  {0x4f, 30000, 1001, 20, 18, 2},
+  {0x5f, 30, 1, 20, 18, 2},
+  {0x6f, 50, 1, 12, 11, 1},
+  {0x7f, 60000, 1001, 10, 9, 1},
+  {0x8f, 60, 1, 10, 9, 1},
+};
+static const struct cdp_fps_entry null_fps_entry = { 0, 0, 0, 0 };
+
+static const struct cdp_fps_entry *
+cdp_fps_entry_from_fps (guint fps_n, guint fps_d)
+{
+  int i;
+  for (i = 0; i < G_N_ELEMENTS (cdp_fps_table); i++) {
+    if (cdp_fps_table[i].fps_n == fps_n && cdp_fps_table[i].fps_d == fps_d)
+      return &cdp_fps_table[i];
+  }
+  return &null_fps_entry;
+}
+
+
+static GstBuffer *
+make_cdp (GstCCCombiner * self, const guint8 * cc_data, guint cc_data_len,
+    const struct cdp_fps_entry *fps_entry, const GstVideoTimeCode * tc)
+{
+  GstByteWriter bw;
+  guint8 flags, checksum;
+  guint i, len;
+  GstBuffer *ret = gst_buffer_new_allocate (NULL, MAX_CDP_PACKET_LEN, NULL);
+  GstMapInfo map;
+
+  gst_buffer_map (ret, &map, GST_MAP_WRITE);
+
+  gst_byte_writer_init_with_data (&bw, map.data, MAX_CDP_PACKET_LEN, FALSE);
+  gst_byte_writer_put_uint16_be_unchecked (&bw, 0x9669);
+  /* Write a length of 0 for now */
+  gst_byte_writer_put_uint8_unchecked (&bw, 0);
+
+  gst_byte_writer_put_uint8_unchecked (&bw, fps_entry->fps_idx);
+
+  /* caption_service_active */
+  flags = 0x02;
+
+  /* ccdata_present */
+  flags |= 0x40;
+
+  if (tc && tc->config.fps_n > 0)
+    flags |= 0x80;
+
+  /* reserved */
+  flags |= 0x01;
+
+  gst_byte_writer_put_uint8_unchecked (&bw, flags);
+
+  gst_byte_writer_put_uint16_be_unchecked (&bw, self->cdp_hdr_sequence_cntr);
+
+  if (tc && tc->config.fps_n > 0) {
+    guint8 u8;
+
+    gst_byte_writer_put_uint8_unchecked (&bw, 0x71);
+    /* reserved 11 - 2 bits */
+    u8 = 0xc0;
+    /* tens of hours - 2 bits */
+    u8 |= ((tc->hours / 10) & 0x3) << 4;
+    /* units of hours - 4 bits */
+    u8 |= (tc->hours % 10) & 0xf;
+    gst_byte_writer_put_uint8_unchecked (&bw, u8);
+
+    /* reserved 1 - 1 bit */
+    u8 = 0x80;
+    /* tens of minutes - 3 bits */
+    u8 |= ((tc->minutes / 10) & 0x7) << 4;
+    /* units of minutes - 4 bits */
+    u8 |= (tc->minutes % 10) & 0xf;
+    gst_byte_writer_put_uint8_unchecked (&bw, u8);
+
+    /* field flag - 1 bit */
+    u8 = tc->field_count < 2 ? 0x00 : 0x80;
+    /* tens of seconds - 3 bits */
+    u8 |= ((tc->seconds / 10) & 0x7) << 4;
+    /* units of seconds - 4 bits */
+    u8 |= (tc->seconds % 10) & 0xf;
+    gst_byte_writer_put_uint8_unchecked (&bw, u8);
+
+    /* drop frame flag - 1 bit */
+    u8 = (tc->config.flags & GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME) ? 0x80 :
+        0x00;
+    /* reserved0 - 1 bit */
+    /* tens of frames - 2 bits */
+    u8 |= ((tc->frames / 10) & 0x3) << 4;
+    /* units of frames 4 bits */
+    u8 |= (tc->frames % 10) & 0xf;
+    gst_byte_writer_put_uint8_unchecked (&bw, u8);
+  }
+
+  gst_byte_writer_put_uint8_unchecked (&bw, 0x72);
+  gst_byte_writer_put_uint8_unchecked (&bw, 0xe0 | fps_entry->max_cc_count);
+  gst_byte_writer_put_data_unchecked (&bw, cc_data, cc_data_len);
+  while (fps_entry->max_cc_count > cc_data_len / 3) {
+    gst_byte_writer_put_uint8_unchecked (&bw, 0xfa);
+    gst_byte_writer_put_uint8_unchecked (&bw, 0x00);
+    gst_byte_writer_put_uint8_unchecked (&bw, 0x00);
+    cc_data_len += 3;
+  }
+
+  gst_byte_writer_put_uint8_unchecked (&bw, 0x74);
+  gst_byte_writer_put_uint16_be_unchecked (&bw, self->cdp_hdr_sequence_cntr);
+  self->cdp_hdr_sequence_cntr++;
+  /* We calculate the checksum afterwards */
+  gst_byte_writer_put_uint8_unchecked (&bw, 0);
+
+  len = gst_byte_writer_get_pos (&bw);
+  gst_byte_writer_set_pos (&bw, 2);
+  gst_byte_writer_put_uint8_unchecked (&bw, len);
+
+  checksum = 0;
+  for (i = 0; i < len; i++) {
+    checksum += map.data[i];
+  }
+  checksum &= 0xff;
+  checksum = 256 - checksum;
+  map.data[len - 1] = checksum;
+
+  gst_buffer_unmap (ret, &map);
+
+  gst_buffer_set_size (ret, len);
+
+  return ret;
+}
+
+static GstBuffer *
+make_padding (GstCCCombiner * self, const GstVideoTimeCode * tc, guint field)
+{
+  GstBuffer *ret = NULL;
+
+  switch (self->caption_type) {
+    case GST_VIDEO_CAPTION_TYPE_CEA708_CDP:
+    {
+      const guint8 cc_data[6] = { 0xf8, 0x80, 0x80, 0xf9, 0x80, 0x80 };
+
+      ret = make_cdp (self, cc_data, 6, self->cdp_fps_entry, tc);
+      break;
+    }
+    case GST_VIDEO_CAPTION_TYPE_CEA708_RAW:
+    {
+      GstMapInfo map;
+
+      ret = gst_buffer_new_allocate (NULL, 3, NULL);
+
+      gst_buffer_map (ret, &map, GST_MAP_WRITE);
+
+      map.data[0] = 0xfc | (field & 0x01);
+      map.data[1] = 0x80;
+      map.data[2] = 0x80;
+
+      gst_buffer_unmap (ret, &map);
+      break;
+    }
+    case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A:
+    {
+      GstMapInfo map;
+
+      ret = gst_buffer_new_allocate (NULL, 3, NULL);
+
+      gst_buffer_map (ret, &map, GST_MAP_WRITE);
+
+      map.data[0] = 0x80 | (field == 0 ? 0x01 : 0x00);
+      map.data[1] = 0x80;
+      map.data[2] = 0x80;
+
+      gst_buffer_unmap (ret, &map);
+      break;
+    }
+    case GST_VIDEO_CAPTION_TYPE_CEA608_RAW:
+    {
+      GstMapInfo map;
+
+      ret = gst_buffer_new_allocate (NULL, 2, NULL);
+
+      gst_buffer_map (ret, &map, GST_MAP_WRITE);
+
+      map.data[0] = 0x80;
+      map.data[1] = 0x80;
+
+      gst_buffer_unmap (ret, &map);
+      break;
+    }
+    default:
+      break;
+  }
+
+  return ret;
+}
+
+static void
+queue_caption (GstCCCombiner * self, GstBuffer * scheduled, guint field)
+{
+  GstAggregatorPad *caption_pad;
+  CaptionQueueItem item;
+
+  if (self->progressive && field == 1) {
+    gst_buffer_unref (scheduled);
+    return;
+  }
+
+  caption_pad =
+      GST_AGGREGATOR_PAD_CAST (gst_element_get_static_pad (GST_ELEMENT_CAST
+          (self), "caption"));
+
+  g_assert (gst_queue_array_get_length (self->scheduled[field]) <=
+      self->max_scheduled);
+
+  if (gst_queue_array_get_length (self->scheduled[field]) ==
+      self->max_scheduled) {
+    CaptionQueueItem *dropped =
+        gst_queue_array_pop_tail_struct (self->scheduled[field]);
+
+    GST_WARNING_OBJECT (self,
+        "scheduled queue runs too long, dropping %" GST_PTR_FORMAT, dropped);
+
+    gst_element_post_message (GST_ELEMENT_CAST (self),
+        gst_message_new_qos (GST_OBJECT_CAST (self), FALSE,
+            dropped->running_time, dropped->stream_time,
+            GST_BUFFER_PTS (dropped->buffer), GST_BUFFER_DURATION (dropped)));
+
+    gst_buffer_unref (dropped->buffer);
+  }
+
+  gst_object_unref (caption_pad);
+
+  item.buffer = scheduled;
+  item.running_time =
+      gst_segment_to_running_time (&caption_pad->segment, GST_FORMAT_TIME,
+      GST_BUFFER_PTS (scheduled));
+  item.stream_time =
+      gst_segment_to_stream_time (&caption_pad->segment, GST_FORMAT_TIME,
+      GST_BUFFER_PTS (scheduled));
+
+  gst_queue_array_push_tail_struct (self->scheduled[field], &item);
+}
+
+static void
+schedule_cdp (GstCCCombiner * self, const GstVideoTimeCode * tc,
+    const guint8 * data, guint len, GstClockTime pts, GstClockTime duration)
+{
+  const guint8 *cc_data;
+  guint cc_data_len;
+  gboolean inject = FALSE;
+
+  if ((cc_data = extract_cdp (data, len, &cc_data_len))) {
+    guint8 i;
+
+    for (i = 0; i < cc_data_len / 3; i++) {
+      gboolean cc_valid = (cc_data[i * 3] & 0x04) == 0x04;
+      guint8 cc_type = cc_data[i * 3] & 0x03;
+
+      if (!cc_valid)
+        break;
+
+      if (cc_type == 0x00 || cc_type == 0x01) {
+        if (cc_data[i * 3 + 1] != 0x80 || cc_data[i * 3 + 2] != 0x80) {
+          inject = TRUE;
+          break;
+        }
+        continue;
+      } else {
+        inject = TRUE;
+        break;
+      }
+    }
+  }
+
+  if (inject) {
+    GstBuffer *buf =
+        make_cdp (self, cc_data, cc_data_len, self->cdp_fps_entry, tc);
+
+    /* We only set those for QoS reporting purposes */
+    GST_BUFFER_PTS (buf) = pts;
+    GST_BUFFER_DURATION (buf) = duration;
+
+    queue_caption (self, buf, 0);
+  }
+}
+
+static void
+schedule_cea608_s334_1a (GstCCCombiner * self, guint8 * data, guint len,
+    GstClockTime pts, GstClockTime duration)
+{
+  guint8 field0_data[3], field1_data[3];
+  guint field0_len = 0, field1_len = 0;
+  guint i;
+  gboolean field0_608 = FALSE, field1_608 = FALSE;
+
+  if (len % 3 != 0) {
+    GST_WARNING ("Invalid cc_data buffer size %u. Truncating to a multiple "
+        "of 3", len);
+    len = len - (len % 3);
+  }
+
+  for (i = 0; i < len / 3; i++) {
+    guint8 cc_type = data[i * 3] & 0x03;
+
+    if (cc_type == 0x01) {
+      if (field0_608)
+        continue;
+
+      field0_608 = TRUE;
+
+      if (data[i * 3 + 1] == 0x80 && data[i * 3 + 2] == 0x80)
+        continue;
+
+      field0_data[field0_len++] = data[i * 3];
+      field0_data[field0_len++] = data[i * 3 + 1];
+      field0_data[field0_len++] = data[i * 3 + 2];
+    } else if (cc_type == 0x00) {
+      if (field1_608)
+        continue;
+
+      field1_608 = TRUE;
+
+      if (data[i * 3 + 1] == 0x80 && data[i * 3 + 2] == 0x80)
+        continue;
+
+      field1_data[field1_len++] = data[i * 3];
+      field1_data[field1_len++] = data[i * 3 + 1];
+      field1_data[field1_len++] = data[i * 3 + 2];
+    } else {
+      break;
+    }
+  }
+
+  if (field0_len > 0) {
+    GstBuffer *buf = gst_buffer_new_allocate (NULL, field0_len, NULL);
+
+    gst_buffer_fill (buf, 0, field0_data, field0_len);
+    GST_BUFFER_PTS (buf) = pts;
+    GST_BUFFER_DURATION (buf) = duration;
+
+    queue_caption (self, buf, 0);
+  }
+
+  if (field1_len > 0) {
+    GstBuffer *buf = gst_buffer_new_allocate (NULL, field1_len, NULL);
+
+    gst_buffer_fill (buf, 0, field1_data, field1_len);
+    GST_BUFFER_PTS (buf) = pts;
+    GST_BUFFER_DURATION (buf) = duration;
+
+    queue_caption (self, buf, 1);
+  }
+}
+
+static void
+schedule_cea708_raw (GstCCCombiner * self, guint8 * data, guint len,
+    GstClockTime pts, GstClockTime duration)
+{
+  guint8 field0_data[MAX_CDP_PACKET_LEN], field1_data[3];
+  guint field0_len = 0, field1_len = 0;
+  guint i;
+  gboolean field0_608 = FALSE, field1_608 = FALSE;
+  gboolean started_ccp = FALSE;
+
+  if (len % 3 != 0) {
+    GST_WARNING ("Invalid cc_data buffer size %u. Truncating to a multiple "
+        "of 3", len);
+    len = len - (len % 3);
+  }
+
+  for (i = 0; i < len / 3; i++) {
+    gboolean cc_valid = (data[i * 3] & 0x04) == 0x04;
+    guint8 cc_type = data[i * 3] & 0x03;
+
+    if (!started_ccp) {
+      if (cc_type == 0x00) {
+        if (!cc_valid)
+          continue;
+
+        if (field0_608)
+          continue;
+
+        field0_608 = TRUE;
+
+        if (data[i * 3 + 1] == 0x80 && data[i * 3 + 2] == 0x80)
+          continue;
+
+        field0_data[field0_len++] = data[i * 3];
+        field0_data[field0_len++] = data[i * 3 + 1];
+        field0_data[field0_len++] = data[i * 3 + 2];
+      } else if (cc_type == 0x01) {
+        if (!cc_valid)
+          continue;
+
+        if (field1_608)
+          continue;
+
+        field1_608 = TRUE;
+
+        if (data[i * 3 + 1] == 0x80 && data[i * 3 + 2] == 0x80)
+          continue;
+
+        field1_data[field1_len++] = data[i * 3];
+        field1_data[field1_len++] = data[i * 3 + 1];
+        field1_data[field1_len++] = data[i * 3 + 2];
+      }
+
+      continue;
+    }
+
+    if (cc_type & 0x10)
+      started_ccp = TRUE;
+
+    if (!cc_valid)
+      continue;
+
+    if (cc_type == 0x00 || cc_type == 0x01)
+      continue;
+
+    field0_data[field0_len++] = data[i * 3];
+    field0_data[field0_len++] = data[i * 3 + 1];
+    field0_data[field0_len++] = data[i * 3 + 2];
+  }
+
+  if (field0_len > 0) {
+    GstBuffer *buf = gst_buffer_new_allocate (NULL, field0_len, NULL);
+
+    gst_buffer_fill (buf, 0, field0_data, field0_len);
+    GST_BUFFER_PTS (buf) = pts;
+    GST_BUFFER_DURATION (buf) = duration;
+
+    queue_caption (self, buf, 0);
+  }
+
+  if (field1_len > 0) {
+    GstBuffer *buf = gst_buffer_new_allocate (NULL, field1_len, NULL);
+
+    gst_buffer_fill (buf, 0, field1_data, field1_len);
+    GST_BUFFER_PTS (buf) = pts;
+    GST_BUFFER_DURATION (buf) = duration;
+
+    queue_caption (self, buf, 1);
+  }
+}
+
+static void
+schedule_cea608_raw (GstCCCombiner * self, guint8 * data, guint len,
+    GstBuffer * buffer)
+{
+  if (len < 2) {
+    return;
+  }
+
+  if (data[0] != 0x80 || data[1] != 0x80) {
+    queue_caption (self, gst_buffer_ref (buffer), 0);
+  }
+}
+
+
+static void
+schedule_caption (GstCCCombiner * self, GstBuffer * caption_buf,
+    const GstVideoTimeCode * tc)
+{
+  GstMapInfo map;
+  GstClockTime pts, duration;
+
+  pts = GST_BUFFER_PTS (caption_buf);
+  duration = GST_BUFFER_DURATION (caption_buf);
+
+  gst_buffer_map (caption_buf, &map, GST_MAP_READ);
+
+  switch (self->caption_type) {
+    case GST_VIDEO_CAPTION_TYPE_CEA708_CDP:
+      schedule_cdp (self, tc, map.data, map.size, pts, duration);
+      break;
+    case GST_VIDEO_CAPTION_TYPE_CEA708_RAW:
+      schedule_cea708_raw (self, map.data, map.size, pts, duration);
+      break;
+    case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A:
+      schedule_cea608_s334_1a (self, map.data, map.size, pts, duration);
+      break;
+    case GST_VIDEO_CAPTION_TYPE_CEA608_RAW:
+      schedule_cea608_raw (self, map.data, map.size, caption_buf);
+      break;
+    default:
+      break;
+  }
+
+  gst_buffer_unmap (caption_buf, &map);
+}
+
+static void
+dequeue_caption (GstCCCombiner * self, const GstVideoTimeCode * tc, guint field)
+{
+  CaptionQueueItem *scheduled;
+  CaptionData caption_data;
+
+  if ((scheduled = gst_queue_array_pop_head_struct (self->scheduled[field]))) {
+    caption_data.buffer = scheduled->buffer;
+    caption_data.caption_type = self->caption_type;
+    g_array_append_val (self->current_frame_captions, caption_data);
+  } else {
+    caption_data.caption_type = self->caption_type;
+    caption_data.buffer = make_padding (self, tc, field);
+    g_array_append_val (self->current_frame_captions, caption_data);
+  }
+}
+
 static GstFlowReturn
 gst_cc_combiner_collect_captions (GstCCCombiner * self, gboolean timeout)
 {
@@ -86,13 +704,14 @@ gst_cc_combiner_collect_captions (GstCCCombiner * self, gboolean timeout)
       GST_AGGREGATOR_PAD (GST_AGGREGATOR_SRC_PAD (self));
   GstAggregatorPad *caption_pad;
   GstBuffer *video_buf;
+  GstVideoTimeCodeMeta *tc_meta;
+  GstVideoTimeCode *tc = NULL;
 
   g_assert (self->current_video_buffer != NULL);
 
   caption_pad =
       GST_AGGREGATOR_PAD_CAST (gst_element_get_static_pad (GST_ELEMENT_CAST
           (self), "caption"));
-
   /* No caption pad, forward buffer directly */
   if (!caption_pad) {
     GST_LOG_OBJECT (self, "No caption pad, passing through video");
@@ -104,6 +723,12 @@ gst_cc_combiner_collect_captions (GstCCCombiner * self, gboolean timeout)
     goto done;
   }
 
+  tc_meta = gst_buffer_get_video_time_code_meta (self->current_video_buffer);
+
+  if (tc_meta) {
+    tc = &tc_meta->tc;
+  }
+
   GST_LOG_OBJECT (self, "Trying to collect captions for queued video buffer");
   do {
     GstBuffer *caption_buf;
@@ -178,44 +803,111 @@ gst_cc_combiner_collect_captions (GstCCCombiner * self, gboolean timeout)
     if (caption_time >= self->current_video_running_time_end) {
       gst_buffer_unref (caption_buf);
       break;
-    } else if (GST_CLOCK_TIME_IS_VALID (self->previous_video_running_time_end)) {
-      if (caption_time < self->previous_video_running_time_end) {
+    } else if (!self->schedule) {
+      if (GST_CLOCK_TIME_IS_VALID (self->previous_video_running_time_end)) {
+        if (caption_time < self->previous_video_running_time_end) {
+          GST_WARNING_OBJECT (self,
+              "Caption buffer before end of last video frame, dropping");
+
+          gst_aggregator_pad_drop_buffer (caption_pad);
+          gst_buffer_unref (caption_buf);
+          continue;
+        }
+      } else if (caption_time < self->current_video_running_time) {
         GST_WARNING_OBJECT (self,
-            "Caption buffer before end of last video frame, dropping");
+            "Caption buffer before current video frame, dropping");
 
         gst_aggregator_pad_drop_buffer (caption_pad);
         gst_buffer_unref (caption_buf);
         continue;
       }
-    } else if (caption_time < self->current_video_running_time) {
-      GST_WARNING_OBJECT (self,
-          "Caption buffer before current video frame, dropping");
-
-      gst_aggregator_pad_drop_buffer (caption_pad);
-      gst_buffer_unref (caption_buf);
-      continue;
     }
 
     /* This caption buffer has to be collected */
     GST_LOG_OBJECT (self,
         "Collecting caption buffer %p %" GST_TIME_FORMAT " for video buffer %p",
         caption_buf, GST_TIME_ARGS (caption_time), self->current_video_buffer);
-    caption_data.caption_type = self->current_caption_type;
-    caption_data.buffer = caption_buf;
-    g_array_append_val (self->current_frame_captions, caption_data);
+
+    caption_data.caption_type = self->caption_type;
+
     gst_aggregator_pad_drop_buffer (caption_pad);
+
+    if (!self->schedule) {
+      caption_data.buffer = caption_buf;
+      g_array_append_val (self->current_frame_captions, caption_data);
+    } else {
+      schedule_caption (self, caption_buf, tc);
+      gst_buffer_unref (caption_buf);
+    }
   } while (TRUE);
 
+  /* FIXME pad correctly according to fps */
+  if (self->schedule) {
+    g_assert (self->current_frame_captions->len == 0);
+
+    switch (self->caption_type) {
+      case GST_VIDEO_CAPTION_TYPE_CEA708_CDP:
+      {
+        /* Only relevant in alternate and mixed mode, no need to look at the caps */
+        if (GST_BUFFER_FLAG_IS_SET (self->current_video_buffer,
+                GST_VIDEO_BUFFER_FLAG_INTERLACED)) {
+          if (GST_VIDEO_BUFFER_IS_TOP_FIELD (self->current_video_buffer)) {
+            dequeue_caption (self, tc, 0);
+          }
+        } else {
+          dequeue_caption (self, tc, 0);
+        }
+        break;
+      }
+      case GST_VIDEO_CAPTION_TYPE_CEA708_RAW:
+      case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A:
+      {
+        if (self->progressive) {
+          dequeue_caption (self, tc, 0);
+        } else if (GST_BUFFER_FLAG_IS_SET (self->current_video_buffer,
+                GST_VIDEO_BUFFER_FLAG_INTERLACED)) {
+          if (GST_VIDEO_BUFFER_IS_TOP_FIELD (self->current_video_buffer)) {
+            dequeue_caption (self, tc, 0);
+          }
+          if (GST_VIDEO_BUFFER_IS_BOTTOM_FIELD (self->current_video_buffer)) {
+            dequeue_caption (self, tc, 1);
+          }
+        } else {
+          dequeue_caption (self, tc, 0);
+          dequeue_caption (self, tc, 1);
+        }
+        break;
+      }
+      case GST_VIDEO_CAPTION_TYPE_CEA608_RAW:
+      {
+        if (self->progressive) {
+          dequeue_caption (self, tc, 0);
+        } else if (GST_BUFFER_FLAG_IS_SET (self->current_video_buffer,
+                GST_VIDEO_BUFFER_FLAG_INTERLACED)) {
+          if (GST_VIDEO_BUFFER_IS_TOP_FIELD (self->current_video_buffer)) {
+            dequeue_caption (self, tc, 0);
+          }
+        } else {
+          dequeue_caption (self, tc, 0);
+        }
+        break;
+      }
+      default:
+        break;
+    }
+  }
+
   gst_aggregator_selected_samples (GST_AGGREGATOR_CAST (self),
       GST_BUFFER_PTS (self->current_video_buffer),
       GST_BUFFER_DTS (self->current_video_buffer),
       GST_BUFFER_DURATION (self->current_video_buffer), NULL);
 
+  GST_LOG_OBJECT (self, "Attaching %u captions to buffer %p",
+      self->current_frame_captions->len, self->current_video_buffer);
+
   if (self->current_frame_captions->len > 0) {
     guint i;
 
-    GST_LOG_OBJECT (self, "Attaching %u captions to buffer %p",
-        self->current_frame_captions->len, self->current_video_buffer);
     video_buf = gst_buffer_make_writable (self->current_video_buffer);
     self->current_video_buffer = NULL;
 
@@ -400,14 +1092,32 @@ gst_cc_combiner_sink_event (GstAggregator * aggregator,
       s = gst_caps_get_structure (caps, 0);
 
       if (strcmp (GST_OBJECT_NAME (agg_pad), "caption") == 0) {
-        self->current_caption_type = gst_video_caption_type_from_caps (caps);
+        GstVideoCaptionType caption_type =
+            gst_video_caption_type_from_caps (caps);
+
+        if (self->caption_type != GST_VIDEO_CAPTION_TYPE_UNKNOWN &&
+            caption_type != self->caption_type) {
+          GST_ERROR_OBJECT (self, "Changing caption type is not allowed");
+
+          GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, (NULL),
+              ("Changing caption type is not allowed"));
+
+          return FALSE;
+        }
+        self->caption_type = caption_type;
       } else {
         gint fps_n, fps_d;
+        const gchar *interlace_mode;
 
         fps_n = fps_d = 0;
 
         gst_structure_get_fraction (s, "framerate", &fps_n, &fps_d);
 
+        interlace_mode = gst_structure_get_string (s, "interlace-mode");
+
+        self->progressive = !interlace_mode
+            || !g_strcmp0 (interlace_mode, "progressive");
+
         if (fps_n != self->video_fps_n || fps_d != self->video_fps_d) {
           GstClockTime latency;
 
@@ -418,6 +1128,8 @@ gst_cc_combiner_sink_event (GstAggregator * aggregator,
         self->video_fps_n = fps_n;
         self->video_fps_d = fps_d;
 
+        self->cdp_fps_entry = cdp_fps_entry_from_fps (fps_n, fps_d);
+
         gst_aggregator_set_src_caps (aggregator, caps);
       }
 
@@ -451,7 +1163,11 @@ gst_cc_combiner_stop (GstAggregator * aggregator)
   gst_buffer_replace (&self->current_video_buffer, NULL);
 
   g_array_set_size (self->current_frame_captions, 0);
-  self->current_caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
+  self->caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
+
+  gst_queue_array_clear (self->scheduled[0]);
+  gst_queue_array_clear (self->scheduled[1]);
+  self->cdp_fps_entry = &null_fps_entry;
 
   return TRUE;
 }
@@ -471,6 +1187,10 @@ gst_cc_combiner_flush (GstAggregator * aggregator)
 
   src_pad->segment.position = GST_CLOCK_TIME_NONE;
 
+  self->cdp_hdr_sequence_cntr = 0;
+  gst_queue_array_clear (self->scheduled[0]);
+  gst_queue_array_clear (self->scheduled[1]);
+
   return GST_FLOW_OK;
 }
 
@@ -493,7 +1213,7 @@ gst_cc_combiner_create_new_pad (GstAggregator * aggregator,
   GST_OBJECT_LOCK (self);
   agg_pad = g_object_new (GST_TYPE_AGGREGATOR_PAD,
       "name", "caption", "direction", GST_PAD_SINK, "template", templ, NULL);
-  self->current_caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
+  self->caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
   GST_OBJECT_UNLOCK (self);
 
   return agg_pad;
@@ -655,6 +1375,61 @@ gst_cc_combiner_peek_next_sample (GstAggregator * agg,
   return res;
 }
 
+static GstStateChangeReturn
+gst_cc_combiner_change_state (GstElement * element, GstStateChange transition)
+{
+  GstCCCombiner *self = GST_CCCOMBINER (element);
+
+  switch (transition) {
+    case GST_STATE_CHANGE_READY_TO_PAUSED:
+      self->schedule = self->prop_schedule;
+      self->max_scheduled = self->prop_max_scheduled;
+      break;
+    default:
+      break;
+  }
+
+  return GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
+}
+
+static void
+gst_cc_combiner_set_property (GObject * object, guint prop_id,
+    const GValue * value, GParamSpec * pspec)
+{
+  GstCCCombiner *self = GST_CCCOMBINER (object);
+
+  switch (prop_id) {
+    case PROP_SCHEDULE:
+      self->prop_schedule = g_value_get_boolean (value);
+      break;
+    case PROP_MAX_SCHEDULED:
+      self->prop_max_scheduled = g_value_get_uint (value);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_cc_combiner_get_property (GObject * object, guint prop_id, GValue * value,
+    GParamSpec * pspec)
+{
+  GstCCCombiner *self = GST_CCCOMBINER (object);
+
+  switch (prop_id) {
+    case PROP_SCHEDULE:
+      g_value_set_boolean (value, self->prop_schedule);
+      break;
+    case PROP_MAX_SCHEDULED:
+      g_value_set_uint (value, self->prop_max_scheduled);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
 static void
 gst_cc_combiner_class_init (GstCCCombinerClass * klass)
 {
@@ -667,6 +1442,8 @@ gst_cc_combiner_class_init (GstCCCombinerClass * klass)
   aggregator_class = (GstAggregatorClass *) klass;
 
   gobject_class->finalize = gst_cc_combiner_finalize;
+  gobject_class->set_property = gst_cc_combiner_set_property;
+  gobject_class->get_property = gst_cc_combiner_get_property;
 
   gst_element_class_set_static_metadata (gstelement_class,
       "Closed Caption Combiner",
@@ -674,6 +1451,56 @@ gst_cc_combiner_class_init (GstCCCombinerClass * klass)
       "Combines GstVideoCaptionMeta with video input stream",
       "Sebastian Dröge <sebastian@centricular.com>");
 
+  /**
+   * GstCCCombiner:schedule:
+   *
+   * Controls whether caption buffers should be smoothly scheduled
+   * in order to have exactly one per output video buffer.
+   *
+   * This can involve rewriting input captions, for example when the
+   * input is CDP sequence counters are rewritten, time codes are dropped
+   * and potentially re-injected if the input video frame had a time code
+   * meta.
+   *
+   * Caption buffers may also get split up in order to assign captions to
+   * the correct field when the input is interlaced.
+   *
+   * This can also imply that the input will drift from synchronization,
+   * when there isn't enough padding in the input stream to catch up. In
+   * that case the element will start dropping old caption buffers once
+   * the number of buffers in its internal queue reaches
+   * #GstCCCombiner:max-scheduled.
+   *
+   * When this is set to %FALSE, the behaviour of this element is essentially
+   * that of a funnel.
+   *
+   * Since: 1.20
+   */
+  g_object_class_install_property (G_OBJECT_CLASS (klass),
+      PROP_SCHEDULE, g_param_spec_boolean ("schedule",
+          "Schedule",
+          "Schedule caption buffers so that exactly one is output per video frame",
+          FALSE,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
+          GST_PARAM_MUTABLE_READY));
+
+  /**
+   * GstCCCombiner:max-scheduled:
+   *
+   * Controls the number of scheduled buffers after which the element
+   * will start dropping old buffers from its internal queues. See
+   * #GstCCCombiner:schedule.
+   *
+   * Since: 1.20
+   */
+  g_object_class_install_property (G_OBJECT_CLASS (klass),
+      PROP_SCHEDULE, g_param_spec_uint ("max-scheduled",
+          "Max Scheduled",
+          "Maximum number of buffers to queue for scheduling", 0, G_MAXUINT,
+          DEFAULT_MAX_SCHEDULED,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS |
+          GST_PARAM_MUTABLE_READY));
+
   gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
       &sinktemplate, GST_TYPE_AGGREGATOR_PAD);
   gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
@@ -681,6 +1508,9 @@ gst_cc_combiner_class_init (GstCCCombinerClass * klass)
   gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
       &captiontemplate, GST_TYPE_AGGREGATOR_PAD);
 
+  gstelement_class->change_state =
+      GST_DEBUG_FUNCPTR (gst_cc_combiner_change_state);
+
   aggregator_class->aggregate = gst_cc_combiner_aggregate;
   aggregator_class->stop = gst_cc_combiner_stop;
   aggregator_class->flush = gst_cc_combiner_flush;
@@ -716,5 +1546,18 @@ gst_cc_combiner_init (GstCCCombiner * self)
   self->current_video_running_time = self->current_video_running_time_end =
       self->previous_video_running_time_end = GST_CLOCK_TIME_NONE;
 
-  self->current_caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
+  self->caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
+
+  self->prop_schedule = DEFAULT_SCHEDULE;
+  self->prop_max_scheduled = DEFAULT_MAX_SCHEDULED;
+  self->scheduled[0] =
+      gst_queue_array_new_for_struct (sizeof (CaptionQueueItem), 0);
+  self->scheduled[1] =
+      gst_queue_array_new_for_struct (sizeof (CaptionQueueItem), 0);
+  gst_queue_array_set_clear_func (self->scheduled[0],
+      (GDestroyNotify) clear_scheduled);
+  gst_queue_array_set_clear_func (self->scheduled[1],
+      (GDestroyNotify) clear_scheduled);
+  self->cdp_hdr_sequence_cntr = 0;
+  self->cdp_fps_entry = &null_fps_entry;
 }
diff --git a/ext/closedcaption/gstcccombiner.h b/ext/closedcaption/gstcccombiner.h
index 93f3d85fa..2965915f3 100644
--- a/ext/closedcaption/gstcccombiner.h
+++ b/ext/closedcaption/gstcccombiner.h
@@ -40,18 +40,38 @@ G_BEGIN_DECLS
 typedef struct _GstCCCombiner GstCCCombiner;
 typedef struct _GstCCCombinerClass GstCCCombinerClass;
 
+struct cdp_fps_entry
+{
+  guint8 fps_idx;
+  guint fps_n, fps_d;
+  guint max_cc_count;
+  guint max_ccp_count;
+  guint max_cea608_count;
+};
+
 struct _GstCCCombiner
 {
   GstAggregator parent;
 
   gint video_fps_n, video_fps_d;
+  gboolean progressive;
   GstClockTime previous_video_running_time_end;
   GstClockTime current_video_running_time;
   GstClockTime current_video_running_time_end;
   GstBuffer *current_video_buffer;
 
   GArray *current_frame_captions;
-  GstVideoCaptionType current_caption_type;
+  GstVideoCaptionType caption_type;
+
+  gboolean prop_schedule;
+  guint prop_max_scheduled;
+
+  gboolean schedule;
+  guint max_scheduled;
+  /* One queue per field */
+  GstQueueArray *scheduled[2];
+  guint16 cdp_hdr_sequence_cntr;
+  const struct cdp_fps_entry *cdp_fps_entry;
 };
 
 struct _GstCCCombinerClass
diff --git a/tests/check/elements/cccombiner.c b/tests/check/elements/cccombiner.c
index 863da0df0..390b984bb 100644
--- a/tests/check/elements/cccombiner.c
+++ b/tests/check/elements/cccombiner.c
@@ -31,8 +31,6 @@
 static GstStaticCaps foo_bar_caps = GST_STATIC_CAPS ("foo/bar");
 static GstStaticCaps cea708_cc_data_caps =
 GST_STATIC_CAPS ("closedcaption/x-cea-708,format=(string) cc_data");
-static GstStaticCaps cea708_cdp_caps =
-GST_STATIC_CAPS ("closedcaption/x-cea-708,format=(string) cdp");
 
 GST_START_TEST (no_captions)
 {
@@ -91,7 +89,6 @@ samples_selected_cb (GstAggregator * agg, GstSegment * segment,
 
   buflist = gst_sample_get_buffer_list (captions_sample);
   fail_unless_equals_int (gst_buffer_list_length (buflist), 1);
-  fail_unless (gst_buffer_list_get (buflist, 0) == expected_caption_buffer);
   gst_sample_unref (captions_sample);
 
   gst_object_unref (caption_pad);
@@ -106,6 +103,7 @@ GST_START_TEST (captions_and_eos)
   GstCaps *caps;
   GstVideoCaptionMeta *meta;
   GstBuffer *second_video_buf, *second_caption_buf;
+  const guint8 cc_data[3] = { 0x0, 0x0, 0x0 };
 
   h = gst_harness_new_with_padnames ("cccombiner", "sink", "src");
   h2 = gst_harness_new_with_element (h->element, NULL, NULL);
@@ -127,7 +125,8 @@ GST_START_TEST (captions_and_eos)
   expected_video_buffer = buf;
   gst_harness_push (h, buf);
 
-  buf = gst_buffer_new_and_alloc (128);
+  buf = gst_buffer_new_and_alloc (3);
+  gst_buffer_fill (buf, 0, cc_data, 3);
   GST_BUFFER_PTS (buf) = 0;
   GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND;
   expected_caption_buffer = buf;
@@ -141,7 +140,8 @@ GST_START_TEST (captions_and_eos)
   second_video_buf = buf;
   gst_harness_push (h, buf);
 
-  buf = gst_buffer_new_and_alloc (128);
+  buf = gst_buffer_new_and_alloc (3);
+  gst_buffer_fill (buf, 0, cc_data, 3);
   GST_BUFFER_PTS (buf) = 40 * GST_MSECOND;
   GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND;
   second_caption_buf = buf;
@@ -158,7 +158,7 @@ GST_START_TEST (captions_and_eos)
   fail_unless (meta != NULL);
   fail_unless_equals_int (meta->caption_type,
       GST_VIDEO_CAPTION_TYPE_CEA708_RAW);
-  fail_unless_equals_int (meta->size, 128);
+  fail_unless_equals_int (meta->size, 3);
 
   gst_buffer_unref (outbuf);
 
@@ -174,91 +174,7 @@ GST_START_TEST (captions_and_eos)
   fail_unless (meta != NULL);
   fail_unless_equals_int (meta->caption_type,
       GST_VIDEO_CAPTION_TYPE_CEA708_RAW);
-  fail_unless_equals_int (meta->size, 128);
-
-  gst_buffer_unref (outbuf);
-
-  /* Caps should be equal to input caps */
-  caps = gst_pad_get_current_caps (h->sinkpad);
-  fail_unless (caps != NULL);
-  fail_unless (gst_caps_can_intersect (caps,
-          gst_static_caps_get (&foo_bar_caps)));
-  gst_caps_unref (caps);
-
-  gst_harness_teardown (h);
-  gst_harness_teardown (h2);
-}
-
-GST_END_TEST;
-
-GST_START_TEST (captions_type_change_and_eos)
-{
-  GstHarness *h, *h2;
-  GstBuffer *buf, *outbuf;
-  GstPad *caption_pad;
-  GstCaps *caps;
-  GstVideoCaptionMeta *meta;
-
-  h = gst_harness_new_with_padnames ("cccombiner", "sink", "src");
-  h2 = gst_harness_new_with_element (h->element, NULL, NULL);
-  caption_pad = gst_element_get_request_pad (h->element, "caption");
-  gst_harness_add_element_sink_pad (h2, caption_pad);
-  gst_object_unref (caption_pad);
-
-  gst_harness_set_src_caps_str (h, foo_bar_caps.string);
-  gst_harness_set_src_caps_str (h2, cea708_cc_data_caps.string);
-
-  /* Push a buffer and caption buffer */
-  buf = gst_buffer_new_and_alloc (128);
-  GST_BUFFER_PTS (buf) = 0;
-  GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND;
-  gst_harness_push (h, buf);
-
-  buf = gst_buffer_new_and_alloc (128);
-  GST_BUFFER_PTS (buf) = 0;
-  GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND;
-  gst_harness_push (h2, buf);
-
-  /* Change caption type */
-  gst_harness_set_src_caps_str (h2, cea708_cdp_caps.string);
-
-  /* And another one: the first video buffer should be retrievable
-   * after the second caption buffer is pushed */
-  buf = gst_buffer_new_and_alloc (128);
-  GST_BUFFER_PTS (buf) = 40 * GST_MSECOND;
-  GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND;
-  gst_harness_push (h, buf);
-
-  buf = gst_buffer_new_and_alloc (128);
-  GST_BUFFER_PTS (buf) = 40 * GST_MSECOND;
-  GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND;
-  gst_harness_push (h2, buf);
-
-  /* Pull the first output buffer */
-  outbuf = gst_harness_pull (h);
-  fail_unless (outbuf != NULL);
-
-  meta = gst_buffer_get_video_caption_meta (outbuf);
-  fail_unless (meta != NULL);
-  fail_unless_equals_int (meta->caption_type,
-      GST_VIDEO_CAPTION_TYPE_CEA708_RAW);
-  fail_unless_equals_int (meta->size, 128);
-
-  gst_buffer_unref (outbuf);
-
-  /* Push EOS on both pads get the second output buffer, we otherwise wait
-   * in case there are further captions for the current video buffer */
-  gst_harness_push_event (h, gst_event_new_eos ());
-  gst_harness_push_event (h2, gst_event_new_eos ());
-
-  outbuf = gst_harness_pull (h);
-  fail_unless (outbuf != NULL);
-
-  meta = gst_buffer_get_video_caption_meta (outbuf);
-  fail_unless (meta != NULL);
-  fail_unless_equals_int (meta->caption_type,
-      GST_VIDEO_CAPTION_TYPE_CEA708_CDP);
-  fail_unless_equals_int (meta->size, 128);
+  fail_unless_equals_int (meta->size, 3);
 
   gst_buffer_unref (outbuf);
 
@@ -285,7 +201,6 @@ cccombiner_suite (void)
 
   tcase_add_test (tc, no_captions);
   tcase_add_test (tc, captions_and_eos);
-  tcase_add_test (tc, captions_type_change_and_eos);
 
   return s;
 }