diff options
| -rw-r--r-- | docs/plugins/gst_plugins_cache.json | 29 | ||||
| -rw-r--r-- | ext/closedcaption/gstcccombiner.c | 883 | ||||
| -rw-r--r-- | ext/closedcaption/gstcccombiner.h | 22 | ||||
| -rw-r--r-- | tests/check/elements/cccombiner.c | 99 |
4 files changed, 919 insertions, 114 deletions
diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 9f995d555..6eae76468 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -3318,7 +3318,34 @@ "type": "GstAggregatorPad" } }, - "properties": {}, + "properties": { + "max-scheduled": { + "blurb": "Maximum number of buffers to queue for scheduling", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "-1", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint", + "writable": true + }, + "schedule": { + "blurb": "Schedule caption buffers so that exactly one is output per video frame", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "true", + "mutable": "ready", + "readable": true, + "type": "gboolean", + "writable": true + } + }, "rank": "none" }, "ccconverter": { diff --git a/ext/closedcaption/gstcccombiner.c b/ext/closedcaption/gstcccombiner.c index f1781c6cb..5e47318f5 100644 --- a/ext/closedcaption/gstcccombiner.c +++ b/ext/closedcaption/gstcccombiner.c @@ -54,12 +54,29 @@ static GstStaticPadTemplate captiontemplate = G_DEFINE_TYPE (GstCCCombiner, gst_cc_combiner, GST_TYPE_AGGREGATOR); #define parent_class gst_cc_combiner_parent_class +enum +{ + PROP_0, + PROP_SCHEDULE, + PROP_MAX_SCHEDULED, +}; + +#define DEFAULT_MAX_SCHEDULED 30 +#define DEFAULT_SCHEDULE TRUE + typedef struct { GstVideoCaptionType caption_type; GstBuffer *buffer; } CaptionData; +typedef struct +{ + GstBuffer *buffer; + GstClockTime running_time; + GstClockTime stream_time; +} CaptionQueueItem; + static void caption_data_clear (CaptionData * data) { @@ -67,10 +84,18 @@ caption_data_clear (CaptionData * data) } static void +clear_scheduled (CaptionQueueItem * item) +{ + gst_buffer_unref (item->buffer); +} + +static void gst_cc_combiner_finalize (GObject * object) { GstCCCombiner *self = GST_CCCOMBINER (object); + gst_queue_array_free (self->scheduled[0]); + gst_queue_array_free (self->scheduled[1]); g_array_unref (self->current_frame_captions); self->current_frame_captions = NULL; @@ -79,6 +104,599 @@ gst_cc_combiner_finalize (GObject * object) #define GST_FLOW_NEED_DATA GST_FLOW_CUSTOM_SUCCESS +static const guint8 * +extract_cdp (const guint8 * cdp, guint cdp_len, guint * cc_data_len) +{ + GstByteReader br; + guint16 u16; + guint8 u8; + guint8 flags; + guint len = 0; + const guint8 *cc_data = NULL; + + *cc_data_len = 0; + + /* Header + footer length */ + if (cdp_len < 11) { + goto done; + } + + gst_byte_reader_init (&br, cdp, cdp_len); + u16 = gst_byte_reader_get_uint16_be_unchecked (&br); + if (u16 != 0x9669) { + goto done; + } + + u8 = gst_byte_reader_get_uint8_unchecked (&br); + if (u8 != cdp_len) { + goto done; + } + + gst_byte_reader_skip_unchecked (&br, 1); + + flags = gst_byte_reader_get_uint8_unchecked (&br); + + /* No cc_data? */ + if ((flags & 0x40) == 0) { + goto done; + } + + /* cdp_hdr_sequence_cntr */ + gst_byte_reader_skip_unchecked (&br, 2); + + /* time_code_present */ + if (flags & 0x80) { + if (gst_byte_reader_get_remaining (&br) < 5) { + goto done; + } + gst_byte_reader_skip_unchecked (&br, 5); + } + + /* ccdata_present */ + if (flags & 0x40) { + guint8 cc_count; + + if (gst_byte_reader_get_remaining (&br) < 2) { + goto done; + } + u8 = gst_byte_reader_get_uint8_unchecked (&br); + if (u8 != 0x72) { + goto done; + } + + cc_count = gst_byte_reader_get_uint8_unchecked (&br); + if ((cc_count & 0xe0) != 0xe0) { + goto done; + } + cc_count &= 0x1f; + + if (cc_count == 0) + return 0; + + len = 3 * cc_count; + if (gst_byte_reader_get_remaining (&br) < len) + goto done; + + cc_data = gst_byte_reader_get_data_unchecked (&br, len); + *cc_data_len = len; + } + +done: + return cc_data; +} + +#define MAX_CDP_PACKET_LEN 256 +#define MAX_CEA608_LEN 32 + +static const struct cdp_fps_entry cdp_fps_table[] = { + {0x1f, 24000, 1001, 25, 22, 3 /* FIXME: alternating max cea608 count! */ }, + {0x2f, 24, 1, 25, 22, 2}, + {0x3f, 25, 1, 24, 22, 2}, + {0x4f, 30000, 1001, 20, 18, 2}, + {0x5f, 30, 1, 20, 18, 2}, + {0x6f, 50, 1, 12, 11, 1}, + {0x7f, 60000, 1001, 10, 9, 1}, + {0x8f, 60, 1, 10, 9, 1}, +}; +static const struct cdp_fps_entry null_fps_entry = { 0, 0, 0, 0 }; + +static const struct cdp_fps_entry * +cdp_fps_entry_from_fps (guint fps_n, guint fps_d) +{ + int i; + for (i = 0; i < G_N_ELEMENTS (cdp_fps_table); i++) { + if (cdp_fps_table[i].fps_n == fps_n && cdp_fps_table[i].fps_d == fps_d) + return &cdp_fps_table[i]; + } + return &null_fps_entry; +} + + +static GstBuffer * +make_cdp (GstCCCombiner * self, const guint8 * cc_data, guint cc_data_len, + const struct cdp_fps_entry *fps_entry, const GstVideoTimeCode * tc) +{ + GstByteWriter bw; + guint8 flags, checksum; + guint i, len; + GstBuffer *ret = gst_buffer_new_allocate (NULL, MAX_CDP_PACKET_LEN, NULL); + GstMapInfo map; + + gst_buffer_map (ret, &map, GST_MAP_WRITE); + + gst_byte_writer_init_with_data (&bw, map.data, MAX_CDP_PACKET_LEN, FALSE); + gst_byte_writer_put_uint16_be_unchecked (&bw, 0x9669); + /* Write a length of 0 for now */ + gst_byte_writer_put_uint8_unchecked (&bw, 0); + + gst_byte_writer_put_uint8_unchecked (&bw, fps_entry->fps_idx); + + /* caption_service_active */ + flags = 0x02; + + /* ccdata_present */ + flags |= 0x40; + + if (tc && tc->config.fps_n > 0) + flags |= 0x80; + + /* reserved */ + flags |= 0x01; + + gst_byte_writer_put_uint8_unchecked (&bw, flags); + + gst_byte_writer_put_uint16_be_unchecked (&bw, self->cdp_hdr_sequence_cntr); + + if (tc && tc->config.fps_n > 0) { + guint8 u8; + + gst_byte_writer_put_uint8_unchecked (&bw, 0x71); + /* reserved 11 - 2 bits */ + u8 = 0xc0; + /* tens of hours - 2 bits */ + u8 |= ((tc->hours / 10) & 0x3) << 4; + /* units of hours - 4 bits */ + u8 |= (tc->hours % 10) & 0xf; + gst_byte_writer_put_uint8_unchecked (&bw, u8); + + /* reserved 1 - 1 bit */ + u8 = 0x80; + /* tens of minutes - 3 bits */ + u8 |= ((tc->minutes / 10) & 0x7) << 4; + /* units of minutes - 4 bits */ + u8 |= (tc->minutes % 10) & 0xf; + gst_byte_writer_put_uint8_unchecked (&bw, u8); + + /* field flag - 1 bit */ + u8 = tc->field_count < 2 ? 0x00 : 0x80; + /* tens of seconds - 3 bits */ + u8 |= ((tc->seconds / 10) & 0x7) << 4; + /* units of seconds - 4 bits */ + u8 |= (tc->seconds % 10) & 0xf; + gst_byte_writer_put_uint8_unchecked (&bw, u8); + + /* drop frame flag - 1 bit */ + u8 = (tc->config.flags & GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME) ? 0x80 : + 0x00; + /* reserved0 - 1 bit */ + /* tens of frames - 2 bits */ + u8 |= ((tc->frames / 10) & 0x3) << 4; + /* units of frames 4 bits */ + u8 |= (tc->frames % 10) & 0xf; + gst_byte_writer_put_uint8_unchecked (&bw, u8); + } + + gst_byte_writer_put_uint8_unchecked (&bw, 0x72); + gst_byte_writer_put_uint8_unchecked (&bw, 0xe0 | fps_entry->max_cc_count); + gst_byte_writer_put_data_unchecked (&bw, cc_data, cc_data_len); + while (fps_entry->max_cc_count > cc_data_len / 3) { + gst_byte_writer_put_uint8_unchecked (&bw, 0xfa); + gst_byte_writer_put_uint8_unchecked (&bw, 0x00); + gst_byte_writer_put_uint8_unchecked (&bw, 0x00); + cc_data_len += 3; + } + + gst_byte_writer_put_uint8_unchecked (&bw, 0x74); + gst_byte_writer_put_uint16_be_unchecked (&bw, self->cdp_hdr_sequence_cntr); + self->cdp_hdr_sequence_cntr++; + /* We calculate the checksum afterwards */ + gst_byte_writer_put_uint8_unchecked (&bw, 0); + + len = gst_byte_writer_get_pos (&bw); + gst_byte_writer_set_pos (&bw, 2); + gst_byte_writer_put_uint8_unchecked (&bw, len); + + checksum = 0; + for (i = 0; i < len; i++) { + checksum += map.data[i]; + } + checksum &= 0xff; + checksum = 256 - checksum; + map.data[len - 1] = checksum; + + gst_buffer_unmap (ret, &map); + + gst_buffer_set_size (ret, len); + + return ret; +} + +static GstBuffer * +make_padding (GstCCCombiner * self, const GstVideoTimeCode * tc, guint field) +{ + GstBuffer *ret = NULL; + + switch (self->caption_type) { + case GST_VIDEO_CAPTION_TYPE_CEA708_CDP: + { + const guint8 cc_data[6] = { 0xf8, 0x80, 0x80, 0xf9, 0x80, 0x80 }; + + ret = make_cdp (self, cc_data, 6, self->cdp_fps_entry, tc); + break; + } + case GST_VIDEO_CAPTION_TYPE_CEA708_RAW: + { + GstMapInfo map; + + ret = gst_buffer_new_allocate (NULL, 3, NULL); + + gst_buffer_map (ret, &map, GST_MAP_WRITE); + + map.data[0] = 0xfc | (field & 0x01); + map.data[1] = 0x80; + map.data[2] = 0x80; + + gst_buffer_unmap (ret, &map); + break; + } + case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A: + { + GstMapInfo map; + + ret = gst_buffer_new_allocate (NULL, 3, NULL); + + gst_buffer_map (ret, &map, GST_MAP_WRITE); + + map.data[0] = 0x80 | (field == 0 ? 0x01 : 0x00); + map.data[1] = 0x80; + map.data[2] = 0x80; + + gst_buffer_unmap (ret, &map); + break; + } + case GST_VIDEO_CAPTION_TYPE_CEA608_RAW: + { + GstMapInfo map; + + ret = gst_buffer_new_allocate (NULL, 2, NULL); + + gst_buffer_map (ret, &map, GST_MAP_WRITE); + + map.data[0] = 0x80; + map.data[1] = 0x80; + + gst_buffer_unmap (ret, &map); + break; + } + default: + break; + } + + return ret; +} + +static void +queue_caption (GstCCCombiner * self, GstBuffer * scheduled, guint field) +{ + GstAggregatorPad *caption_pad; + CaptionQueueItem item; + + if (self->progressive && field == 1) { + gst_buffer_unref (scheduled); + return; + } + + caption_pad = + GST_AGGREGATOR_PAD_CAST (gst_element_get_static_pad (GST_ELEMENT_CAST + (self), "caption")); + + g_assert (gst_queue_array_get_length (self->scheduled[field]) <= + self->max_scheduled); + + if (gst_queue_array_get_length (self->scheduled[field]) == + self->max_scheduled) { + CaptionQueueItem *dropped = + gst_queue_array_pop_tail_struct (self->scheduled[field]); + + GST_WARNING_OBJECT (self, + "scheduled queue runs too long, dropping %" GST_PTR_FORMAT, dropped); + + gst_element_post_message (GST_ELEMENT_CAST (self), + gst_message_new_qos (GST_OBJECT_CAST (self), FALSE, + dropped->running_time, dropped->stream_time, + GST_BUFFER_PTS (dropped->buffer), GST_BUFFER_DURATION (dropped))); + + gst_buffer_unref (dropped->buffer); + } + + gst_object_unref (caption_pad); + + item.buffer = scheduled; + item.running_time = + gst_segment_to_running_time (&caption_pad->segment, GST_FORMAT_TIME, + GST_BUFFER_PTS (scheduled)); + item.stream_time = + gst_segment_to_stream_time (&caption_pad->segment, GST_FORMAT_TIME, + GST_BUFFER_PTS (scheduled)); + + gst_queue_array_push_tail_struct (self->scheduled[field], &item); +} + +static void +schedule_cdp (GstCCCombiner * self, const GstVideoTimeCode * tc, + const guint8 * data, guint len, GstClockTime pts, GstClockTime duration) +{ + const guint8 *cc_data; + guint cc_data_len; + gboolean inject = FALSE; + + if ((cc_data = extract_cdp (data, len, &cc_data_len))) { + guint8 i; + + for (i = 0; i < cc_data_len / 3; i++) { + gboolean cc_valid = (cc_data[i * 3] & 0x04) == 0x04; + guint8 cc_type = cc_data[i * 3] & 0x03; + + if (!cc_valid) + break; + + if (cc_type == 0x00 || cc_type == 0x01) { + if (cc_data[i * 3 + 1] != 0x80 || cc_data[i * 3 + 2] != 0x80) { + inject = TRUE; + break; + } + continue; + } else { + inject = TRUE; + break; + } + } + } + + if (inject) { + GstBuffer *buf = + make_cdp (self, cc_data, cc_data_len, self->cdp_fps_entry, tc); + + /* We only set those for QoS reporting purposes */ + GST_BUFFER_PTS (buf) = pts; + GST_BUFFER_DURATION (buf) = duration; + + queue_caption (self, buf, 0); + } +} + +static void +schedule_cea608_s334_1a (GstCCCombiner * self, guint8 * data, guint len, + GstClockTime pts, GstClockTime duration) +{ + guint8 field0_data[3], field1_data[3]; + guint field0_len = 0, field1_len = 0; + guint i; + gboolean field0_608 = FALSE, field1_608 = FALSE; + + if (len % 3 != 0) { + GST_WARNING ("Invalid cc_data buffer size %u. Truncating to a multiple " + "of 3", len); + len = len - (len % 3); + } + + for (i = 0; i < len / 3; i++) { + guint8 cc_type = data[i * 3] & 0x03; + + if (cc_type == 0x01) { + if (field0_608) + continue; + + field0_608 = TRUE; + + if (data[i * 3 + 1] == 0x80 && data[i * 3 + 2] == 0x80) + continue; + + field0_data[field0_len++] = data[i * 3]; + field0_data[field0_len++] = data[i * 3 + 1]; + field0_data[field0_len++] = data[i * 3 + 2]; + } else if (cc_type == 0x00) { + if (field1_608) + continue; + + field1_608 = TRUE; + + if (data[i * 3 + 1] == 0x80 && data[i * 3 + 2] == 0x80) + continue; + + field1_data[field1_len++] = data[i * 3]; + field1_data[field1_len++] = data[i * 3 + 1]; + field1_data[field1_len++] = data[i * 3 + 2]; + } else { + break; + } + } + + if (field0_len > 0) { + GstBuffer *buf = gst_buffer_new_allocate (NULL, field0_len, NULL); + + gst_buffer_fill (buf, 0, field0_data, field0_len); + GST_BUFFER_PTS (buf) = pts; + GST_BUFFER_DURATION (buf) = duration; + + queue_caption (self, buf, 0); + } + + if (field1_len > 0) { + GstBuffer *buf = gst_buffer_new_allocate (NULL, field1_len, NULL); + + gst_buffer_fill (buf, 0, field1_data, field1_len); + GST_BUFFER_PTS (buf) = pts; + GST_BUFFER_DURATION (buf) = duration; + + queue_caption (self, buf, 1); + } +} + +static void +schedule_cea708_raw (GstCCCombiner * self, guint8 * data, guint len, + GstClockTime pts, GstClockTime duration) +{ + guint8 field0_data[MAX_CDP_PACKET_LEN], field1_data[3]; + guint field0_len = 0, field1_len = 0; + guint i; + gboolean field0_608 = FALSE, field1_608 = FALSE; + gboolean started_ccp = FALSE; + + if (len % 3 != 0) { + GST_WARNING ("Invalid cc_data buffer size %u. Truncating to a multiple " + "of 3", len); + len = len - (len % 3); + } + + for (i = 0; i < len / 3; i++) { + gboolean cc_valid = (data[i * 3] & 0x04) == 0x04; + guint8 cc_type = data[i * 3] & 0x03; + + if (!started_ccp) { + if (cc_type == 0x00) { + if (!cc_valid) + continue; + + if (field0_608) + continue; + + field0_608 = TRUE; + + if (data[i * 3 + 1] == 0x80 && data[i * 3 + 2] == 0x80) + continue; + + field0_data[field0_len++] = data[i * 3]; + field0_data[field0_len++] = data[i * 3 + 1]; + field0_data[field0_len++] = data[i * 3 + 2]; + } else if (cc_type == 0x01) { + if (!cc_valid) + continue; + + if (field1_608) + continue; + + field1_608 = TRUE; + + if (data[i * 3 + 1] == 0x80 && data[i * 3 + 2] == 0x80) + continue; + + field1_data[field1_len++] = data[i * 3]; + field1_data[field1_len++] = data[i * 3 + 1]; + field1_data[field1_len++] = data[i * 3 + 2]; + } + + continue; + } + + if (cc_type & 0x10) + started_ccp = TRUE; + + if (!cc_valid) + continue; + + if (cc_type == 0x00 || cc_type == 0x01) + continue; + + field0_data[field0_len++] = data[i * 3]; + field0_data[field0_len++] = data[i * 3 + 1]; + field0_data[field0_len++] = data[i * 3 + 2]; + } + + if (field0_len > 0) { + GstBuffer *buf = gst_buffer_new_allocate (NULL, field0_len, NULL); + + gst_buffer_fill (buf, 0, field0_data, field0_len); + GST_BUFFER_PTS (buf) = pts; + GST_BUFFER_DURATION (buf) = duration; + + queue_caption (self, buf, 0); + } + + if (field1_len > 0) { + GstBuffer *buf = gst_buffer_new_allocate (NULL, field1_len, NULL); + + gst_buffer_fill (buf, 0, field1_data, field1_len); + GST_BUFFER_PTS (buf) = pts; + GST_BUFFER_DURATION (buf) = duration; + + queue_caption (self, buf, 1); + } +} + +static void +schedule_cea608_raw (GstCCCombiner * self, guint8 * data, guint len, + GstBuffer * buffer) +{ + if (len < 2) { + return; + } + + if (data[0] != 0x80 || data[1] != 0x80) { + queue_caption (self, gst_buffer_ref (buffer), 0); + } +} + + +static void +schedule_caption (GstCCCombiner * self, GstBuffer * caption_buf, + const GstVideoTimeCode * tc) +{ + GstMapInfo map; + GstClockTime pts, duration; + + pts = GST_BUFFER_PTS (caption_buf); + duration = GST_BUFFER_DURATION (caption_buf); + + gst_buffer_map (caption_buf, &map, GST_MAP_READ); + + switch (self->caption_type) { + case GST_VIDEO_CAPTION_TYPE_CEA708_CDP: + schedule_cdp (self, tc, map.data, map.size, pts, duration); + break; + case GST_VIDEO_CAPTION_TYPE_CEA708_RAW: + schedule_cea708_raw (self, map.data, map.size, pts, duration); + break; + case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A: + schedule_cea608_s334_1a (self, map.data, map.size, pts, duration); + break; + case GST_VIDEO_CAPTION_TYPE_CEA608_RAW: + schedule_cea608_raw (self, map.data, map.size, caption_buf); + break; + default: + break; + } + + gst_buffer_unmap (caption_buf, &map); +} + +static void +dequeue_caption (GstCCCombiner * self, const GstVideoTimeCode * tc, guint field) +{ + CaptionQueueItem *scheduled; + CaptionData caption_data; + + if ((scheduled = gst_queue_array_pop_head_struct (self->scheduled[field]))) { + caption_data.buffer = scheduled->buffer; + caption_data.caption_type = self->caption_type; + g_array_append_val (self->current_frame_captions, caption_data); + } else { + caption_data.caption_type = self->caption_type; + caption_data.buffer = make_padding (self, tc, field); + g_array_append_val (self->current_frame_captions, caption_data); + } +} + static GstFlowReturn gst_cc_combiner_collect_captions (GstCCCombiner * self, gboolean timeout) { @@ -86,13 +704,14 @@ gst_cc_combiner_collect_captions (GstCCCombiner * self, gboolean timeout) GST_AGGREGATOR_PAD (GST_AGGREGATOR_SRC_PAD (self)); GstAggregatorPad *caption_pad; GstBuffer *video_buf; + GstVideoTimeCodeMeta *tc_meta; + GstVideoTimeCode *tc = NULL; g_assert (self->current_video_buffer != NULL); caption_pad = GST_AGGREGATOR_PAD_CAST (gst_element_get_static_pad (GST_ELEMENT_CAST (self), "caption")); - /* No caption pad, forward buffer directly */ if (!caption_pad) { GST_LOG_OBJECT (self, "No caption pad, passing through video"); @@ -104,6 +723,12 @@ gst_cc_combiner_collect_captions (GstCCCombiner * self, gboolean timeout) goto done; } + tc_meta = gst_buffer_get_video_time_code_meta (self->current_video_buffer); + + if (tc_meta) { + tc = &tc_meta->tc; + } + GST_LOG_OBJECT (self, "Trying to collect captions for queued video buffer"); do { GstBuffer *caption_buf; @@ -178,44 +803,111 @@ gst_cc_combiner_collect_captions (GstCCCombiner * self, gboolean timeout) if (caption_time >= self->current_video_running_time_end) { gst_buffer_unref (caption_buf); break; - } else if (GST_CLOCK_TIME_IS_VALID (self->previous_video_running_time_end)) { - if (caption_time < self->previous_video_running_time_end) { + } else if (!self->schedule) { + if (GST_CLOCK_TIME_IS_VALID (self->previous_video_running_time_end)) { + if (caption_time < self->previous_video_running_time_end) { + GST_WARNING_OBJECT (self, + "Caption buffer before end of last video frame, dropping"); + + gst_aggregator_pad_drop_buffer (caption_pad); + gst_buffer_unref (caption_buf); + continue; + } + } else if (caption_time < self->current_video_running_time) { GST_WARNING_OBJECT (self, - "Caption buffer before end of last video frame, dropping"); + "Caption buffer before current video frame, dropping"); gst_aggregator_pad_drop_buffer (caption_pad); gst_buffer_unref (caption_buf); continue; } - } else if (caption_time < self->current_video_running_time) { - GST_WARNING_OBJECT (self, - "Caption buffer before current video frame, dropping"); - - gst_aggregator_pad_drop_buffer (caption_pad); - gst_buffer_unref (caption_buf); - continue; } /* This caption buffer has to be collected */ GST_LOG_OBJECT (self, "Collecting caption buffer %p %" GST_TIME_FORMAT " for video buffer %p", caption_buf, GST_TIME_ARGS (caption_time), self->current_video_buffer); - caption_data.caption_type = self->current_caption_type; - caption_data.buffer = caption_buf; - g_array_append_val (self->current_frame_captions, caption_data); + + caption_data.caption_type = self->caption_type; + gst_aggregator_pad_drop_buffer (caption_pad); + + if (!self->schedule) { + caption_data.buffer = caption_buf; + g_array_append_val (self->current_frame_captions, caption_data); + } else { + schedule_caption (self, caption_buf, tc); + gst_buffer_unref (caption_buf); + } } while (TRUE); + /* FIXME pad correctly according to fps */ + if (self->schedule) { + g_assert (self->current_frame_captions->len == 0); + + switch (self->caption_type) { + case GST_VIDEO_CAPTION_TYPE_CEA708_CDP: + { + /* Only relevant in alternate and mixed mode, no need to look at the caps */ + if (GST_BUFFER_FLAG_IS_SET (self->current_video_buffer, + GST_VIDEO_BUFFER_FLAG_INTERLACED)) { + if (GST_VIDEO_BUFFER_IS_TOP_FIELD (self->current_video_buffer)) { + dequeue_caption (self, tc, 0); + } + } else { + dequeue_caption (self, tc, 0); + } + break; + } + case GST_VIDEO_CAPTION_TYPE_CEA708_RAW: + case GST_VIDEO_CAPTION_TYPE_CEA608_S334_1A: + { + if (self->progressive) { + dequeue_caption (self, tc, 0); + } else if (GST_BUFFER_FLAG_IS_SET (self->current_video_buffer, + GST_VIDEO_BUFFER_FLAG_INTERLACED)) { + if (GST_VIDEO_BUFFER_IS_TOP_FIELD (self->current_video_buffer)) { + dequeue_caption (self, tc, 0); + } + if (GST_VIDEO_BUFFER_IS_BOTTOM_FIELD (self->current_video_buffer)) { + dequeue_caption (self, tc, 1); + } + } else { + dequeue_caption (self, tc, 0); + dequeue_caption (self, tc, 1); + } + break; + } + case GST_VIDEO_CAPTION_TYPE_CEA608_RAW: + { + if (self->progressive) { + dequeue_caption (self, tc, 0); + } else if (GST_BUFFER_FLAG_IS_SET (self->current_video_buffer, + GST_VIDEO_BUFFER_FLAG_INTERLACED)) { + if (GST_VIDEO_BUFFER_IS_TOP_FIELD (self->current_video_buffer)) { + dequeue_caption (self, tc, 0); + } + } else { + dequeue_caption (self, tc, 0); + } + break; + } + default: + break; + } + } + gst_aggregator_selected_samples (GST_AGGREGATOR_CAST (self), GST_BUFFER_PTS (self->current_video_buffer), GST_BUFFER_DTS (self->current_video_buffer), GST_BUFFER_DURATION (self->current_video_buffer), NULL); + GST_LOG_OBJECT (self, "Attaching %u captions to buffer %p", + self->current_frame_captions->len, self->current_video_buffer); + if (self->current_frame_captions->len > 0) { guint i; - GST_LOG_OBJECT (self, "Attaching %u captions to buffer %p", - self->current_frame_captions->len, self->current_video_buffer); video_buf = gst_buffer_make_writable (self->current_video_buffer); self->current_video_buffer = NULL; @@ -400,14 +1092,32 @@ gst_cc_combiner_sink_event (GstAggregator * aggregator, s = gst_caps_get_structure (caps, 0); if (strcmp (GST_OBJECT_NAME (agg_pad), "caption") == 0) { - self->current_caption_type = gst_video_caption_type_from_caps (caps); + GstVideoCaptionType caption_type = + gst_video_caption_type_from_caps (caps); + + if (self->caption_type != GST_VIDEO_CAPTION_TYPE_UNKNOWN && + caption_type != self->caption_type) { + GST_ERROR_OBJECT (self, "Changing caption type is not allowed"); + + GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, (NULL), + ("Changing caption type is not allowed")); + + return FALSE; + } + self->caption_type = caption_type; } else { gint fps_n, fps_d; + const gchar *interlace_mode; fps_n = fps_d = 0; gst_structure_get_fraction (s, "framerate", &fps_n, &fps_d); + interlace_mode = gst_structure_get_string (s, "interlace-mode"); + + self->progressive = !interlace_mode + || !g_strcmp0 (interlace_mode, "progressive"); + if (fps_n != self->video_fps_n || fps_d != self->video_fps_d) { GstClockTime latency; @@ -418,6 +1128,8 @@ gst_cc_combiner_sink_event (GstAggregator * aggregator, self->video_fps_n = fps_n; self->video_fps_d = fps_d; + self->cdp_fps_entry = cdp_fps_entry_from_fps (fps_n, fps_d); + gst_aggregator_set_src_caps (aggregator, caps); } @@ -451,7 +1163,11 @@ gst_cc_combiner_stop (GstAggregator * aggregator) gst_buffer_replace (&self->current_video_buffer, NULL); g_array_set_size (self->current_frame_captions, 0); - self->current_caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN; + self->caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN; + + gst_queue_array_clear (self->scheduled[0]); + gst_queue_array_clear (self->scheduled[1]); + self->cdp_fps_entry = &null_fps_entry; return TRUE; } @@ -471,6 +1187,10 @@ gst_cc_combiner_flush (GstAggregator * aggregator) src_pad->segment.position = GST_CLOCK_TIME_NONE; + self->cdp_hdr_sequence_cntr = 0; + gst_queue_array_clear (self->scheduled[0]); + gst_queue_array_clear (self->scheduled[1]); + return GST_FLOW_OK; } @@ -493,7 +1213,7 @@ gst_cc_combiner_create_new_pad (GstAggregator * aggregator, GST_OBJECT_LOCK (self); agg_pad = g_object_new (GST_TYPE_AGGREGATOR_PAD, "name", "caption", "direction", GST_PAD_SINK, "template", templ, NULL); - self->current_caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN; + self->caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN; GST_OBJECT_UNLOCK (self); return agg_pad; @@ -655,6 +1375,61 @@ gst_cc_combiner_peek_next_sample (GstAggregator * agg, return res; } +static GstStateChangeReturn +gst_cc_combiner_change_state (GstElement * element, GstStateChange transition) +{ + GstCCCombiner *self = GST_CCCOMBINER (element); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + self->schedule = self->prop_schedule; + self->max_scheduled = self->prop_max_scheduled; + break; + default: + break; + } + + return GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); +} + +static void +gst_cc_combiner_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstCCCombiner *self = GST_CCCOMBINER (object); + + switch (prop_id) { + case PROP_SCHEDULE: + self->prop_schedule = g_value_get_boolean (value); + break; + case PROP_MAX_SCHEDULED: + self->prop_max_scheduled = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_cc_combiner_get_property (GObject * object, guint prop_id, GValue * value, + GParamSpec * pspec) +{ + GstCCCombiner *self = GST_CCCOMBINER (object); + + switch (prop_id) { + case PROP_SCHEDULE: + g_value_set_boolean (value, self->prop_schedule); + break; + case PROP_MAX_SCHEDULED: + g_value_set_uint (value, self->prop_max_scheduled); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + static void gst_cc_combiner_class_init (GstCCCombinerClass * klass) { @@ -667,6 +1442,8 @@ gst_cc_combiner_class_init (GstCCCombinerClass * klass) aggregator_class = (GstAggregatorClass *) klass; gobject_class->finalize = gst_cc_combiner_finalize; + gobject_class->set_property = gst_cc_combiner_set_property; + gobject_class->get_property = gst_cc_combiner_get_property; gst_element_class_set_static_metadata (gstelement_class, "Closed Caption Combiner", @@ -674,6 +1451,56 @@ gst_cc_combiner_class_init (GstCCCombinerClass * klass) "Combines GstVideoCaptionMeta with video input stream", "Sebastian Dröge <sebastian@centricular.com>"); + /** + * GstCCCombiner:schedule: + * + * Controls whether caption buffers should be smoothly scheduled + * in order to have exactly one per output video buffer. + * + * This can involve rewriting input captions, for example when the + * input is CDP sequence counters are rewritten, time codes are dropped + * and potentially re-injected if the input video frame had a time code + * meta. + * + * Caption buffers may also get split up in order to assign captions to + * the correct field when the input is interlaced. + * + * This can also imply that the input will drift from synchronization, + * when there isn't enough padding in the input stream to catch up. In + * that case the element will start dropping old caption buffers once + * the number of buffers in its internal queue reaches + * #GstCCCombiner:max-scheduled. + * + * When this is set to %FALSE, the behaviour of this element is essentially + * that of a funnel. + * + * Since: 1.20 + */ + g_object_class_install_property (G_OBJECT_CLASS (klass), + PROP_SCHEDULE, g_param_spec_boolean ("schedule", + "Schedule", + "Schedule caption buffers so that exactly one is output per video frame", + FALSE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | + GST_PARAM_MUTABLE_READY)); + + /** + * GstCCCombiner:max-scheduled: + * + * Controls the number of scheduled buffers after which the element + * will start dropping old buffers from its internal queues. See + * #GstCCCombiner:schedule. + * + * Since: 1.20 + */ + g_object_class_install_property (G_OBJECT_CLASS (klass), + PROP_SCHEDULE, g_param_spec_uint ("max-scheduled", + "Max Scheduled", + "Maximum number of buffers to queue for scheduling", 0, G_MAXUINT, + DEFAULT_MAX_SCHEDULED, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | + GST_PARAM_MUTABLE_READY)); + gst_element_class_add_static_pad_template_with_gtype (gstelement_class, &sinktemplate, GST_TYPE_AGGREGATOR_PAD); gst_element_class_add_static_pad_template_with_gtype (gstelement_class, @@ -681,6 +1508,9 @@ gst_cc_combiner_class_init (GstCCCombinerClass * klass) gst_element_class_add_static_pad_template_with_gtype (gstelement_class, &captiontemplate, GST_TYPE_AGGREGATOR_PAD); + gstelement_class->change_state = + GST_DEBUG_FUNCPTR (gst_cc_combiner_change_state); + aggregator_class->aggregate = gst_cc_combiner_aggregate; aggregator_class->stop = gst_cc_combiner_stop; aggregator_class->flush = gst_cc_combiner_flush; @@ -716,5 +1546,18 @@ gst_cc_combiner_init (GstCCCombiner * self) self->current_video_running_time = self->current_video_running_time_end = self->previous_video_running_time_end = GST_CLOCK_TIME_NONE; - self->current_caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN; + self->caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN; + + self->prop_schedule = DEFAULT_SCHEDULE; + self->prop_max_scheduled = DEFAULT_MAX_SCHEDULED; + self->scheduled[0] = + gst_queue_array_new_for_struct (sizeof (CaptionQueueItem), 0); + self->scheduled[1] = + gst_queue_array_new_for_struct (sizeof (CaptionQueueItem), 0); + gst_queue_array_set_clear_func (self->scheduled[0], + (GDestroyNotify) clear_scheduled); + gst_queue_array_set_clear_func (self->scheduled[1], + (GDestroyNotify) clear_scheduled); + self->cdp_hdr_sequence_cntr = 0; + self->cdp_fps_entry = &null_fps_entry; } diff --git a/ext/closedcaption/gstcccombiner.h b/ext/closedcaption/gstcccombiner.h index 93f3d85fa..2965915f3 100644 --- a/ext/closedcaption/gstcccombiner.h +++ b/ext/closedcaption/gstcccombiner.h @@ -40,18 +40,38 @@ G_BEGIN_DECLS typedef struct _GstCCCombiner GstCCCombiner; typedef struct _GstCCCombinerClass GstCCCombinerClass; +struct cdp_fps_entry +{ + guint8 fps_idx; + guint fps_n, fps_d; + guint max_cc_count; + guint max_ccp_count; + guint max_cea608_count; +}; + struct _GstCCCombiner { GstAggregator parent; gint video_fps_n, video_fps_d; + gboolean progressive; GstClockTime previous_video_running_time_end; GstClockTime current_video_running_time; GstClockTime current_video_running_time_end; GstBuffer *current_video_buffer; GArray *current_frame_captions; - GstVideoCaptionType current_caption_type; + GstVideoCaptionType caption_type; + + gboolean prop_schedule; + guint prop_max_scheduled; + + gboolean schedule; + guint max_scheduled; + /* One queue per field */ + GstQueueArray *scheduled[2]; + guint16 cdp_hdr_sequence_cntr; + const struct cdp_fps_entry *cdp_fps_entry; }; struct _GstCCCombinerClass diff --git a/tests/check/elements/cccombiner.c b/tests/check/elements/cccombiner.c index 863da0df0..390b984bb 100644 --- a/tests/check/elements/cccombiner.c +++ b/tests/check/elements/cccombiner.c @@ -31,8 +31,6 @@ static GstStaticCaps foo_bar_caps = GST_STATIC_CAPS ("foo/bar"); static GstStaticCaps cea708_cc_data_caps = GST_STATIC_CAPS ("closedcaption/x-cea-708,format=(string) cc_data"); -static GstStaticCaps cea708_cdp_caps = -GST_STATIC_CAPS ("closedcaption/x-cea-708,format=(string) cdp"); GST_START_TEST (no_captions) { @@ -91,7 +89,6 @@ samples_selected_cb (GstAggregator * agg, GstSegment * segment, buflist = gst_sample_get_buffer_list (captions_sample); fail_unless_equals_int (gst_buffer_list_length (buflist), 1); - fail_unless (gst_buffer_list_get (buflist, 0) == expected_caption_buffer); gst_sample_unref (captions_sample); gst_object_unref (caption_pad); @@ -106,6 +103,7 @@ GST_START_TEST (captions_and_eos) GstCaps *caps; GstVideoCaptionMeta *meta; GstBuffer *second_video_buf, *second_caption_buf; + const guint8 cc_data[3] = { 0x0, 0x0, 0x0 }; h = gst_harness_new_with_padnames ("cccombiner", "sink", "src"); h2 = gst_harness_new_with_element (h->element, NULL, NULL); @@ -127,7 +125,8 @@ GST_START_TEST (captions_and_eos) expected_video_buffer = buf; gst_harness_push (h, buf); - buf = gst_buffer_new_and_alloc (128); + buf = gst_buffer_new_and_alloc (3); + gst_buffer_fill (buf, 0, cc_data, 3); GST_BUFFER_PTS (buf) = 0; GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND; expected_caption_buffer = buf; @@ -141,7 +140,8 @@ GST_START_TEST (captions_and_eos) second_video_buf = buf; gst_harness_push (h, buf); - buf = gst_buffer_new_and_alloc (128); + buf = gst_buffer_new_and_alloc (3); + gst_buffer_fill (buf, 0, cc_data, 3); GST_BUFFER_PTS (buf) = 40 * GST_MSECOND; GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND; second_caption_buf = buf; @@ -158,7 +158,7 @@ GST_START_TEST (captions_and_eos) fail_unless (meta != NULL); fail_unless_equals_int (meta->caption_type, GST_VIDEO_CAPTION_TYPE_CEA708_RAW); - fail_unless_equals_int (meta->size, 128); + fail_unless_equals_int (meta->size, 3); gst_buffer_unref (outbuf); @@ -174,91 +174,7 @@ GST_START_TEST (captions_and_eos) fail_unless (meta != NULL); fail_unless_equals_int (meta->caption_type, GST_VIDEO_CAPTION_TYPE_CEA708_RAW); - fail_unless_equals_int (meta->size, 128); - - gst_buffer_unref (outbuf); - - /* Caps should be equal to input caps */ - caps = gst_pad_get_current_caps (h->sinkpad); - fail_unless (caps != NULL); - fail_unless (gst_caps_can_intersect (caps, - gst_static_caps_get (&foo_bar_caps))); - gst_caps_unref (caps); - - gst_harness_teardown (h); - gst_harness_teardown (h2); -} - -GST_END_TEST; - -GST_START_TEST (captions_type_change_and_eos) -{ - GstHarness *h, *h2; - GstBuffer *buf, *outbuf; - GstPad *caption_pad; - GstCaps *caps; - GstVideoCaptionMeta *meta; - - h = gst_harness_new_with_padnames ("cccombiner", "sink", "src"); - h2 = gst_harness_new_with_element (h->element, NULL, NULL); - caption_pad = gst_element_get_request_pad (h->element, "caption"); - gst_harness_add_element_sink_pad (h2, caption_pad); - gst_object_unref (caption_pad); - - gst_harness_set_src_caps_str (h, foo_bar_caps.string); - gst_harness_set_src_caps_str (h2, cea708_cc_data_caps.string); - - /* Push a buffer and caption buffer */ - buf = gst_buffer_new_and_alloc (128); - GST_BUFFER_PTS (buf) = 0; - GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND; - gst_harness_push (h, buf); - - buf = gst_buffer_new_and_alloc (128); - GST_BUFFER_PTS (buf) = 0; - GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND; - gst_harness_push (h2, buf); - - /* Change caption type */ - gst_harness_set_src_caps_str (h2, cea708_cdp_caps.string); - - /* And another one: the first video buffer should be retrievable - * after the second caption buffer is pushed */ - buf = gst_buffer_new_and_alloc (128); - GST_BUFFER_PTS (buf) = 40 * GST_MSECOND; - GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND; - gst_harness_push (h, buf); - - buf = gst_buffer_new_and_alloc (128); - GST_BUFFER_PTS (buf) = 40 * GST_MSECOND; - GST_BUFFER_DURATION (buf) = 40 * GST_MSECOND; - gst_harness_push (h2, buf); - - /* Pull the first output buffer */ - outbuf = gst_harness_pull (h); - fail_unless (outbuf != NULL); - - meta = gst_buffer_get_video_caption_meta (outbuf); - fail_unless (meta != NULL); - fail_unless_equals_int (meta->caption_type, - GST_VIDEO_CAPTION_TYPE_CEA708_RAW); - fail_unless_equals_int (meta->size, 128); - - gst_buffer_unref (outbuf); - - /* Push EOS on both pads get the second output buffer, we otherwise wait - * in case there are further captions for the current video buffer */ - gst_harness_push_event (h, gst_event_new_eos ()); - gst_harness_push_event (h2, gst_event_new_eos ()); - - outbuf = gst_harness_pull (h); - fail_unless (outbuf != NULL); - - meta = gst_buffer_get_video_caption_meta (outbuf); - fail_unless (meta != NULL); - fail_unless_equals_int (meta->caption_type, - GST_VIDEO_CAPTION_TYPE_CEA708_CDP); - fail_unless_equals_int (meta->size, 128); + fail_unless_equals_int (meta->size, 3); gst_buffer_unref (outbuf); @@ -285,7 +201,6 @@ cccombiner_suite (void) tcase_add_test (tc, no_captions); tcase_add_test (tc, captions_and_eos); - tcase_add_test (tc, captions_type_change_and_eos); return s; } |