From 89bb244ab9fe57645c8da2ac7b267a0469048e4e Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Tue, 6 Oct 2020 03:47:42 +0900 Subject: avaudenc/avvidenc: Reopen encoding session if it's required Since the commit https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/22b25b3ea5c, ffmpeg will not clear draning flag for encoder by avcodec_flush_buffers() API by default. Allowed case is only if encoder has AV_CODEC_CAP_ENCODER_FLUSH capability flag. If it's not supported, we should re-open encoding session, otherwise ffmpeg encoder will keep returning AVERROR_EOF Part-of: --- ext/libav/gstavaudenc.c | 32 ++++++++++++++++++++++++++++++++ ext/libav/gstavaudenc.h | 1 + ext/libav/gstavutils.h | 22 +++++++++++++++++++++- ext/libav/gstavvidenc.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ ext/libav/gstavvidenc.h | 1 + 5 files changed, 99 insertions(+), 1 deletion(-) diff --git a/ext/libav/gstavaudenc.c b/ext/libav/gstavaudenc.c index cfd0300..3ff6432 100644 --- a/ext/libav/gstavaudenc.c +++ b/ext/libav/gstavaudenc.c @@ -190,6 +190,9 @@ gst_ffmpegaudenc_start (GstAudioEncoder * encoder) GstFFMpegAudEncClass *oclass = (GstFFMpegAudEncClass *) G_OBJECT_GET_CLASS (ffmpegaudenc); + ffmpegaudenc->opened = FALSE; + ffmpegaudenc->need_reopen = FALSE; + gst_ffmpeg_avcodec_close (ffmpegaudenc->context); if (avcodec_get_context_defaults3 (ffmpegaudenc->context, oclass->in_plugin) < 0) { @@ -208,6 +211,7 @@ gst_ffmpegaudenc_stop (GstAudioEncoder * encoder) /* close old session */ gst_ffmpeg_avcodec_close (ffmpegaudenc->context); ffmpegaudenc->opened = FALSE; + ffmpegaudenc->need_reopen = FALSE; return TRUE; } @@ -233,6 +237,8 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info) GstFFMpegAudEncClass *oclass = (GstFFMpegAudEncClass *) G_OBJECT_GET_CLASS (ffmpegaudenc); + ffmpegaudenc->need_reopen = FALSE; + /* close old session */ if (ffmpegaudenc->opened) { gst_ffmpeg_avcodec_close (ffmpegaudenc->context); @@ -368,6 +374,7 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info) /* success! */ ffmpegaudenc->opened = TRUE; + ffmpegaudenc->need_reopen = FALSE; return TRUE; } @@ -530,9 +537,21 @@ gst_ffmpegaudenc_send_frame (GstFFMpegAudEnc * ffmpegaudenc, GstBuffer * buffer) av_frame_unref (frame); } else { + GstFFMpegAudEncClass *oclass = + (GstFFMpegAudEncClass *) G_OBJECT_GET_CLASS (ffmpegaudenc); + GST_LOG_OBJECT (ffmpegaudenc, "draining"); /* flushing the encoder */ res = avcodec_send_frame (ctx, NULL); + + /* If AV_CODEC_CAP_ENCODER_FLUSH wasn't set, we need to re-open + * encoder */ + if (!(oclass->in_plugin->capabilities & AV_CODEC_CAP_ENCODER_FLUSH)) { + GST_DEBUG_OBJECT (ffmpegaudenc, "Encoder needs reopen later"); + + /* we will reopen later handle_frame() */ + ffmpegaudenc->need_reopen = TRUE; + } } if (res == 0) { @@ -604,6 +623,7 @@ gst_ffmpegaudenc_drain (GstFFMpegAudEnc * ffmpegaudenc) } while (got_packet); } + /* NOTE: this may or may not work depending on capability */ avcodec_flush_buffers (ffmpegaudenc->context); /* FFMpeg will return AVERROR_EOF if it's internal was fully drained @@ -632,6 +652,18 @@ gst_ffmpegaudenc_handle_frame (GstAudioEncoder * encoder, GstBuffer * inbuf) if (!inbuf) return gst_ffmpegaudenc_drain (ffmpegaudenc); + /* endoder was drained or flushed, and ffmpeg encoder doesn't support + * flushing. We need to re-open encoder then */ + if (ffmpegaudenc->need_reopen) { + GST_DEBUG_OBJECT (ffmpegaudenc, "Open encoder again"); + + if (!gst_ffmpegaudenc_set_format (encoder, + gst_audio_encoder_get_audio_info (encoder))) { + GST_ERROR_OBJECT (ffmpegaudenc, "Couldn't re-open encoder"); + return GST_FLOW_NOT_NEGOTIATED; + } + } + inbuf = gst_buffer_ref (inbuf); GST_DEBUG_OBJECT (ffmpegaudenc, diff --git a/ext/libav/gstavaudenc.h b/ext/libav/gstavaudenc.h index 23640bb..3c94aef 100644 --- a/ext/libav/gstavaudenc.h +++ b/ext/libav/gstavaudenc.h @@ -39,6 +39,7 @@ struct _GstFFMpegAudEnc AVCodecContext *context; AVCodecContext *refcontext; gboolean opened; + gboolean need_reopen; AVFrame *frame; diff --git a/ext/libav/gstavutils.h b/ext/libav/gstavutils.h index d18b979..7b41b66 100644 --- a/ext/libav/gstavutils.h +++ b/ext/libav/gstavutils.h @@ -29,6 +29,26 @@ #include +/* Introduced since ffmpeg version 4.3 + * + * Note: Not all ffmpeg encoders seem to be reusable after flushing/draining. + * So if ffmpeg encoder doesn't support it, we should reopen encoding session. + * + * Before ffmpeg 4.3, avcodec_flush_buffers() was implemented in + * libavcodec/decodec.c but it was moved to libavcodec/utils.c and it would be + * accepted if encoder supports AV_CODEC_CAP_ENCODER_FLUSH flag. + * That implies that avcodec_flush_buffers() wasn't intended to be working + * properly for encoders. + */ +#ifndef AV_CODEC_CAP_ENCODER_FLUSH +/* + * This encoder can be flushed using avcodec_flush_buffers(). If this flag is + * not set, the encoder must be closed and reopened to ensure that no frames + * remain pending. + */ +#define AV_CODEC_CAP_ENCODER_FLUSH (1 << 21) +#endif + /* *Get the size of an picture */ @@ -79,7 +99,7 @@ gst_ffmpeg_time_gst_to_ff (guint64 time, AVRational base) return out; } -void +void gst_ffmpeg_init_pix_fmt_info(void); int diff --git a/ext/libav/gstavvidenc.c b/ext/libav/gstavvidenc.c index 2616e8d..0468d88 100644 --- a/ext/libav/gstavvidenc.c +++ b/ext/libav/gstavvidenc.c @@ -243,6 +243,8 @@ gst_ffmpegvidenc_set_format (GstVideoEncoder * encoder, GstFFMpegVidEncClass *oclass = (GstFFMpegVidEncClass *) G_OBJECT_GET_CLASS (ffmpegenc); + ffmpegenc->need_reopen = FALSE; + /* close old session */ if (ffmpegenc->opened) { gst_ffmpeg_avcodec_close (ffmpegenc->context); @@ -622,6 +624,20 @@ gst_ffmpegvidenc_send_frame (GstFFMpegVidEnc * ffmpegenc, ffmpegenc->context->ticks_per_frame, ffmpegenc->context->time_base); send_frame: + if (!picture) { + GstFFMpegVidEncClass *oclass = + (GstFFMpegVidEncClass *) (G_OBJECT_GET_CLASS (ffmpegenc)); + + /* If AV_CODEC_CAP_ENCODER_FLUSH wasn't set, we need to re-open + * encoder */ + if (!(oclass->in_plugin->capabilities & AV_CODEC_CAP_ENCODER_FLUSH)) { + GST_DEBUG_OBJECT (ffmpegenc, "Encoder needs reopen later"); + + /* we will reopen later handle_frame() */ + ffmpegenc->need_reopen = TRUE; + } + } + res = avcodec_send_frame (ffmpegenc->context, picture); if (picture) @@ -710,6 +726,30 @@ gst_ffmpegvidenc_handle_frame (GstVideoEncoder * encoder, GstFlowReturn ret; gboolean got_packet; + /* endoder was drained or flushed, and ffmpeg encoder doesn't support + * flushing. We need to re-open encoder then */ + if (ffmpegenc->need_reopen) { + gboolean reopen_ret; + GstVideoCodecState *input_state; + + GST_DEBUG_OBJECT (ffmpegenc, "Open encoder again"); + + if (!ffmpegenc->input_state) { + GST_ERROR_OBJECT (ffmpegenc, + "Cannot re-open encoder without input state"); + return GST_FLOW_NOT_NEGOTIATED; + } + + input_state = gst_video_codec_state_ref (ffmpegenc->input_state); + reopen_ret = gst_ffmpegvidenc_set_format (encoder, input_state); + gst_video_codec_state_unref (input_state); + + if (!reopen_ret) { + GST_ERROR_OBJECT (ffmpegenc, "Couldn't re-open encoder"); + return GST_FLOW_NOT_NEGOTIATED; + } + } + ret = gst_ffmpegvidenc_send_frame (ffmpegenc, frame); if (ret != GST_FLOW_OK) @@ -852,6 +892,9 @@ gst_ffmpegvidenc_start (GstVideoEncoder * encoder) GstFFMpegVidEncClass *oclass = (GstFFMpegVidEncClass *) G_OBJECT_GET_CLASS (ffmpegenc); + ffmpegenc->opened = FALSE; + ffmpegenc->need_reopen = FALSE; + /* close old session */ gst_ffmpeg_avcodec_close (ffmpegenc->context); if (avcodec_get_context_defaults3 (ffmpegenc->context, oclass->in_plugin) < 0) { @@ -872,6 +915,7 @@ gst_ffmpegvidenc_stop (GstVideoEncoder * encoder) gst_ffmpegvidenc_flush_buffers (ffmpegenc, FALSE); gst_ffmpeg_avcodec_close (ffmpegenc->context); ffmpegenc->opened = FALSE; + ffmpegenc->need_reopen = FALSE; if (ffmpegenc->input_state) { gst_video_codec_state_unref (ffmpegenc->input_state); diff --git a/ext/libav/gstavvidenc.h b/ext/libav/gstavvidenc.h index d81d72b..2d0b7c8 100644 --- a/ext/libav/gstavvidenc.h +++ b/ext/libav/gstavvidenc.h @@ -41,6 +41,7 @@ struct _GstFFMpegVidEnc AVCodecContext *context; AVFrame *picture; gboolean opened; + gboolean need_reopen; gboolean discont; guint pass; gfloat quantizer; -- cgit v1.2.1