From 4aa72cea4f99c9c47f5d88f294942b8db4a60aa0 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Tue, 10 Aug 2021 02:09:09 +0200 Subject: smartencoder: clean up and extend accepted formats * Add support for H265 * Don't overwrite original codec_data / streamheader in the output caps, but instead allow them to change and send them to the combiner at the right moment: encoder caps, reencoded GOP, original caps, original GOP(s), and potentially encoder caps and rencoded last GOP. * For H264 / H265, force usage of a format with inband SPS / PPS (avc3 / hev1), this is cleaner than misadvertising avc1, hvc1 and some muxers like mp4mux will actually advertise both differently. Unfortunately, while mp4 supports updating the codec_data and using avc1 with no in-band SPS / PPS updates, it turns out some decoders (eg chrome / firefox) don't handle this particularly well and stop decoding after the reencoded GOP. We could expose a switch to force usage of avc1 / hvc1 nevertheless, but for now stick to requiring that the parser output SPS / PPS in-band with config-interval=-1 (that has not changed) Part-of: --- gst/encoding/gstencodebasebin.c | 62 +++++++++++++--- gst/encoding/gstsmartencoder.c | 156 ++++++++++++++++++++++++---------------- gst/encoding/gstsmartencoder.h | 3 +- 3 files changed, 151 insertions(+), 70 deletions(-) diff --git a/gst/encoding/gstencodebasebin.c b/gst/encoding/gstencodebasebin.c index ec9104601..e5a69c8f2 100644 --- a/gst/encoding/gstencodebasebin.c +++ b/gst/encoding/gstencodebasebin.c @@ -1178,7 +1178,6 @@ _capsfilter_force_format (GstPad * pad, GParamSpec * arg G_GNUC_UNUSED, StreamGroup * sgroup) { GstCaps *caps; - GstStructure *structure; GstElement *parent = GST_ELEMENT_CAST (gst_object_get_parent (GST_OBJECT (pad))); @@ -1190,15 +1189,37 @@ _capsfilter_force_format (GstPad * pad, g_object_get (pad, "caps", &caps, NULL); caps = gst_caps_copy (caps); - structure = gst_caps_get_structure (caps, 0); - gst_structure_remove_field (structure, "streamheader"); GST_INFO_OBJECT (pad, "Forcing caps to %" GST_PTR_FORMAT, caps); if (parent == sgroup->outfilter || parent == sgroup->smart_capsfilter) { /* outfilter and the smart encoder internal capsfilter need to always be * in sync so the caps match between the two */ if (sgroup->smart_capsfilter) { - gst_structure_remove_field (structure, "codec_data"); - /* The smart encoder handles codec_data itself */ + GstStructure *structure = gst_caps_get_structure (caps, 0); + + /* Pick a stream format that allows for in-band SPS updates, and remove + * restrictions on fields that can be updated by codec_data or in-band SPS + */ + if (gst_structure_has_name (structure, "video/x-h264")) { + gst_structure_set (structure, "stream-format", + G_TYPE_STRING, "avc3", NULL); + + gst_structure_remove_fields (structure, "codec_data", "profile", + "level", NULL); + } else if (gst_structure_has_name (structure, "video/x-h265")) { + gst_structure_set (structure, "stream-format", + G_TYPE_STRING, "hev1", NULL); + + gst_structure_remove_fields (structure, "codec_data", "tier", "profile", + "level", NULL); + } + + /* For VP8 / VP9, streamheader in the caps is informative, and + * not actually used by muxers, we can allow it to change */ + if (gst_structure_has_name (structure, "video/x-vp8") || + gst_structure_has_name (structure, "video/x-vp9")) { + gst_structure_remove_field (structure, "streamheader"); + } + g_object_set (sgroup->smart_capsfilter, "caps", caps, NULL); g_signal_handler_disconnect (sgroup->smart_capsfilter->sinkpads->data, @@ -1293,10 +1314,12 @@ setup_smart_encoder (GstEncodeBaseBin * ebin, GstEncodingProfile * sprof, GstElement *sinkelement, *convert = NULL; GstElement *smartencoder = g_object_new (GST_TYPE_SMART_ENCODER, NULL); GstPad *srcpad = gst_element_get_static_pad (smartencoder, "src"); - GstCaps *format = gst_encoding_profile_get_format (sprof); + GstCaps *format = + gst_caps_make_writable (gst_encoding_profile_get_format (sprof)); GstCaps *tmpcaps = gst_pad_query_caps (srcpad, NULL); const gboolean native_video = ! !(ebin->flags & GST_ENCODEBIN_FLAG_NO_VIDEO_CONVERSION); + GstStructure *structure = gst_caps_get_structure (format, 0); /* Check if stream format is compatible */ if (!gst_caps_can_intersect (tmpcaps, format)) { @@ -1315,6 +1338,29 @@ setup_smart_encoder (GstEncodeBaseBin * ebin, GstEncodingProfile * sprof, parser = _get_parser (ebin, sprof, encoder); sgroup->smart_capsfilter = gst_element_factory_make ("capsfilter", NULL); reencoder_bin = gst_bin_new (NULL); + + /* Pick a stream format that allows for in-band SPS updates, and remove + * restrictions on fields that can be updated by codec_data or in-band SPS + */ + if (gst_structure_has_name (structure, "video/x-h264")) { + gst_structure_set (structure, "stream-format", G_TYPE_STRING, "avc3", NULL); + + gst_structure_remove_fields (structure, "codec_data", "profile", + "level", NULL); + } else if (gst_structure_has_name (structure, "video/x-h265")) { + gst_structure_set (structure, "stream-format", G_TYPE_STRING, "hev1", NULL); + + gst_structure_remove_fields (structure, "codec_data", "tier", "profile", + "level", NULL); + } + + /* For VP8 / VP9, streamheader in the caps is informative, and + * not actually used by muxers, we can allow it to change */ + if (gst_structure_has_name (structure, "video/x-vp8") || + gst_structure_has_name (structure, "video/x-vp9")) { + gst_structure_remove_field (structure, "streamheader"); + } + g_object_set (sgroup->smart_capsfilter, "caps", format, NULL); gst_bin_add_many (GST_BIN (reencoder_bin), @@ -1564,8 +1610,8 @@ _create_stream_group (GstEncodeBaseBin * ebin, GstEncodingProfile * sprof, /* Expose input queue or identity sink pad as ghostpad */ sinkpad = - gst_element_get_static_pad (sgroup->identity ? sgroup-> - identity : sgroup->inqueue, "sink"); + gst_element_get_static_pad (sgroup->identity ? sgroup->identity : sgroup-> + inqueue, "sink"); if (sinkpadname == NULL) { gchar *pname = g_strdup_printf ("%s_%u", gst_encoding_profile_get_type_nick (sprof), diff --git a/gst/encoding/gstsmartencoder.c b/gst/encoding/gstsmartencoder.c index 473be18d1..b647b8cca 100644 --- a/gst/encoding/gstsmartencoder.c +++ b/gst/encoding/gstsmartencoder.c @@ -34,6 +34,7 @@ GST_DEBUG_CATEGORY_STATIC (smart_encoder_debug); "video/x-vp8;"\ "video/x-vp9;"\ "video/x-h264;"\ + "video/x-h265;"\ "video/mpeg,mpegversion=(int)1,systemstream=(boolean)false;"\ "video/mpeg,mpegversion=(int)2,systemstream=(boolean)false;" @@ -120,50 +121,25 @@ internal_event_func (GstPad * pad, GstObject * parent, GstEvent * event) break; case GST_EVENT_SEGMENT: gst_event_copy_segment (event, &self->internal_segment); - break; - case GST_EVENT_CAPS: - { - GstCaps *caps; - gst_event_parse_caps (event, &caps); - caps = gst_caps_copy (caps); - if (self->last_caps) { - GstBuffer *codec_data = NULL, *stream_header; - GstCaps *new_caps; - GstStructure *last_struct = gst_caps_get_structure (self->last_caps, 0); - - if (gst_structure_get (last_struct, "codec_data", GST_TYPE_BUFFER, - &codec_data, NULL) && codec_data) { - gst_structure_set (gst_caps_get_structure (caps, 0), "codec_data", - GST_TYPE_BUFFER, codec_data, NULL); - } - - if (gst_structure_get (last_struct, "stream_header", GST_TYPE_BUFFER, - &stream_header, NULL) && stream_header) { - gst_structure_set (gst_caps_get_structure (caps, 0), "stream_header", - GST_TYPE_BUFFER, stream_header, NULL); - } + if (self->output_segment.format == GST_FORMAT_UNDEFINED) { + gst_segment_init (&self->output_segment, GST_FORMAT_TIME); - new_caps = gst_caps_intersect (self->last_caps, caps); - if (!new_caps || gst_caps_is_empty (new_caps)) { - GST_ERROR_OBJECT (parent, "New caps from reencoder %" GST_PTR_FORMAT - " are not compatible with previous caps: %" GST_PTR_FORMAT, caps, - self->last_caps); + /* Ensure that we can represent negative DTS in our 'single' segment */ + self->output_segment.start = 60 * 60 * GST_SECOND * 1000; + if (!gst_pad_push_event (self->srcpad, + gst_event_new_segment (&self->output_segment))) { + GST_ERROR_OBJECT (self, "Could not push segment!"); - g_mutex_lock (&self->internal_flow_lock); - self->internal_flow = GST_FLOW_NOT_NEGOTIATED; - g_cond_signal (&self->internal_flow_cond); - g_mutex_unlock (&self->internal_flow_lock); + GST_ELEMENT_FLOW_ERROR (self, GST_FLOW_ERROR); return FALSE; } - - gst_caps_unref (caps); - caps = new_caps; } - event = gst_event_new_caps (caps); - self->last_caps = caps; + break; + case GST_EVENT_CAPS: + { return gst_pad_push_event (self->srcpad, event); } default: @@ -389,21 +365,6 @@ gst_smart_encoder_push_pending_gop (GstSmartEncoder * self) GST_DEBUG ("Pushing pending GOP (%" GST_TIME_FORMAT " -- %" GST_TIME_FORMAT ")", GST_TIME_ARGS (self->gop_start), GST_TIME_ARGS (self->gop_stop)); - if (self->output_segment.format == GST_FORMAT_UNDEFINED) { - gst_segment_init (&self->output_segment, GST_FORMAT_TIME); - - /* Ensure that we can represent negative DTS in our 'single' segment */ - self->output_segment.start = 60 * 60 * GST_SECOND * 1000; - if (!gst_pad_push_event (self->srcpad, - gst_event_new_segment (&self->output_segment))) { - GST_ERROR_OBJECT (self, "Could not push segment!"); - - GST_ELEMENT_FLOW_ERROR (self, GST_FLOW_ERROR); - - return GST_FLOW_ERROR; - } - } - if (!self->pending_gop) { /* This might happen on EOS */ GST_INFO_OBJECT (self, "Empty gop!"); @@ -431,7 +392,31 @@ gst_smart_encoder_push_pending_gop (GstSmartEncoder * self) GST_TIME_FORMAT " - %" GST_SEGMENT_FORMAT, GST_TIME_ARGS (cstart), GST_TIME_ARGS (cstop), &self->input_segment); res = gst_smart_encoder_reencode_gop (self); + + /* Make sure we push the original caps when resuming the original stream */ + self->push_original_caps = TRUE; } else { + if (self->push_original_caps) { + gst_pad_push_event (self->srcpad, + gst_event_new_caps (self->original_caps)); + self->push_original_caps = FALSE; + } + + if (self->output_segment.format == GST_FORMAT_UNDEFINED) { + gst_segment_init (&self->output_segment, GST_FORMAT_TIME); + + /* Ensure that we can represent negative DTS in our 'single' segment */ + self->output_segment.start = 60 * 60 * GST_SECOND * 1000; + if (!gst_pad_push_event (self->srcpad, + gst_event_new_segment (&self->output_segment))) { + GST_ERROR_OBJECT (self, "Could not push segment!"); + + GST_ELEMENT_FLOW_ERROR (self, GST_FLOW_ERROR); + + return GST_FLOW_ERROR; + } + } + /* The whole GOP is within the segment, push all pending buffers downstream */ GST_INFO_OBJECT (self, "GOP doesn't need to be modified, pushing downstream: %" GST_TIME_FORMAT @@ -523,13 +508,18 @@ smart_encoder_sink_event (GstPad * pad, GstObject * ghostpad, GstEvent * event) smart_encoder_reset (self); break; case GST_EVENT_CAPS: - if (self->last_caps) { - gst_clear_event (&event); - } else { - gst_event_parse_caps (event, &self->last_caps); - self->last_caps = gst_caps_copy (self->last_caps); - } + { + GstCaps *caps; + + gst_event_parse_caps (event, &caps); + if (self->original_caps) + gst_caps_unref (self->original_caps); + + self->original_caps = gst_caps_ref (caps); + self->push_original_caps = TRUE; + gst_clear_event (&event); break; + } case GST_EVENT_STREAM_START: gst_event_replace (&self->stream_start_event, gst_event_ref (event)); break; @@ -620,14 +610,30 @@ _pad_sink_acceptcaps (GstPad * pad, GstSmartEncoder * self, GstCaps * caps) n = gst_caps_get_size (accepted_caps); for (i = 0; i < n; i++) { s = gst_caps_get_structure (accepted_caps, i); - gst_structure_remove_fields (s, "codec_data", NULL); + + if (gst_structure_has_name (s, "video/x-h264") || + gst_structure_has_name (s, "video/x-h265")) { + gst_structure_remove_fields (s, "codec_data", "tier", "profile", "level", + NULL); + } else if (gst_structure_has_name (s, "video/x-vp8") + || gst_structure_has_name (s, "video/x-vp9")) { + gst_structure_remove_field (s, "streamheader"); + } } modified_caps = gst_caps_copy (caps); n = gst_caps_get_size (modified_caps); for (i = 0; i < n; i++) { s = gst_caps_get_structure (modified_caps, i); - gst_structure_remove_fields (s, "codec_data", NULL); + + if (gst_structure_has_name (s, "video/x-h264") || + gst_structure_has_name (s, "video/x-h265")) { + gst_structure_remove_fields (s, "codec_data", "tier", "profile", "level", + NULL); + } else if (gst_structure_has_name (s, "video/x-vp8") + || gst_structure_has_name (s, "video/x-vp9")) { + gst_structure_remove_field (s, "streamheader"); + } } ret = gst_caps_can_intersect (modified_caps, accepted_caps); @@ -689,9 +695,32 @@ gst_smart_encoder_add_parser (GstSmartEncoder * self, GstCaps * format) goto failed; } - /* Add SPS/PPS before each gop to ensure that they can be decoded - * independently */ g_object_set (parser, "config-interval", -1, NULL); + + if (!gst_bin_add (GST_BIN (self), parser)) { + GST_ERROR_OBJECT (self, "Could not add parser."); + + goto failed; + } + + if (!gst_element_link (parser, capsfilter)) { + GST_ERROR_OBJECT (self, "Could not link capfilter and parser."); + + goto failed; + } + + sinkpad = gst_element_get_static_pad (parser, "sink"); + } else if (gst_structure_has_name (gst_caps_get_structure (format, 0), + "video/x-h265")) { + GstElement *parser = gst_element_factory_make ("h265parse", NULL); + if (!parser) { + GST_ERROR_OBJECT (self, "`h265parse` is missing, can't encode smartly"); + + goto failed; + } + + g_object_set (parser, "config-interval", -1, NULL); + if (!gst_bin_add (GST_BIN (self), parser)) { GST_ERROR_OBJECT (self, "Could not add parser."); @@ -793,6 +822,11 @@ gst_smart_encoder_dispose (GObject * object) gst_clear_object (&self->encoder); + if (self->original_caps) { + gst_caps_unref (self->original_caps); + self->original_caps = NULL; + } + G_OBJECT_CLASS (gst_smart_encoder_parent_class)->dispose (object); } diff --git a/gst/encoding/gstsmartencoder.h b/gst/encoding/gstsmartencoder.h index 71e79011f..0af29d552 100644 --- a/gst/encoding/gstsmartencoder.h +++ b/gst/encoding/gstsmartencoder.h @@ -45,7 +45,8 @@ struct _GstSmartEncoder { GstSegment internal_segment; GstClockTime last_dts; - GstCaps *last_caps; + GstCaps *original_caps; + gboolean push_original_caps; GstEvent *segment_event; GstEvent *stream_start_event; -- cgit v1.2.1