avaudenc/avvidenc: Reopen encoding session if it's required

Since the commit https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/22b25b3ea5c,
ffmpeg will not clear draning flag for encoder by avcodec_flush_buffers() API
by default. Allowed case is only if encoder has AV_CODEC_CAP_ENCODER_FLUSH
capability flag. If it's not supported, we should re-open encoding
session, otherwise ffmpeg encoder will keep returning AVERROR_EOF

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-libav/-/merge_requests/99>

5 files changed, 99 insertions, 1 deletions

diff --git a/ext/libav/gstavaudenc.c b/ext/libav/gstavaudenc.c
index cfd0300..3ff6432 100644
--- a/ext/libav/gstavaudenc.c
+++ b/ext/libav/gstavaudenc.c
@@ -190,6 +190,9 @@ gst_ffmpegaudenc_start (GstAudioEncoder * encoder)
   GstFFMpegAudEncClass *oclass =
       (GstFFMpegAudEncClass *) G_OBJECT_GET_CLASS (ffmpegaudenc);
 
+  ffmpegaudenc->opened = FALSE;
+  ffmpegaudenc->need_reopen = FALSE;
+
   gst_ffmpeg_avcodec_close (ffmpegaudenc->context);
   if (avcodec_get_context_defaults3 (ffmpegaudenc->context,
           oclass->in_plugin) < 0) {
@@ -208,6 +211,7 @@ gst_ffmpegaudenc_stop (GstAudioEncoder * encoder)
   /* close old session */
   gst_ffmpeg_avcodec_close (ffmpegaudenc->context);
   ffmpegaudenc->opened = FALSE;
+  ffmpegaudenc->need_reopen = FALSE;
 
   return TRUE;
 }
@@ -233,6 +237,8 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info)
   GstFFMpegAudEncClass *oclass =
       (GstFFMpegAudEncClass *) G_OBJECT_GET_CLASS (ffmpegaudenc);
 
+  ffmpegaudenc->need_reopen = FALSE;
+
   /* close old session */
   if (ffmpegaudenc->opened) {
     gst_ffmpeg_avcodec_close (ffmpegaudenc->context);
@@ -368,6 +374,7 @@ gst_ffmpegaudenc_set_format (GstAudioEncoder * encoder, GstAudioInfo * info)
 
   /* success! */
   ffmpegaudenc->opened = TRUE;
+  ffmpegaudenc->need_reopen = FALSE;
 
   return TRUE;
 }
@@ -530,9 +537,21 @@ gst_ffmpegaudenc_send_frame (GstFFMpegAudEnc * ffmpegaudenc, GstBuffer * buffer)
 
     av_frame_unref (frame);
   } else {
+    GstFFMpegAudEncClass *oclass =
+        (GstFFMpegAudEncClass *) G_OBJECT_GET_CLASS (ffmpegaudenc);
+
     GST_LOG_OBJECT (ffmpegaudenc, "draining");
     /* flushing the encoder */
     res = avcodec_send_frame (ctx, NULL);
+
+    /* If AV_CODEC_CAP_ENCODER_FLUSH wasn't set, we need to re-open
+     * encoder */
+    if (!(oclass->in_plugin->capabilities & AV_CODEC_CAP_ENCODER_FLUSH)) {
+      GST_DEBUG_OBJECT (ffmpegaudenc, "Encoder needs reopen later");
+
+      /* we will reopen later handle_frame() */
+      ffmpegaudenc->need_reopen = TRUE;
+    }
   }
 
   if (res == 0) {
@@ -604,6 +623,7 @@ gst_ffmpegaudenc_drain (GstFFMpegAudEnc * ffmpegaudenc)
     } while (got_packet);
   }
 
+  /* NOTE: this may or may not work depending on capability */
   avcodec_flush_buffers (ffmpegaudenc->context);
 
   /* FFMpeg will return AVERROR_EOF if it's internal was fully drained
@@ -632,6 +652,18 @@ gst_ffmpegaudenc_handle_frame (GstAudioEncoder * encoder, GstBuffer * inbuf)
   if (!inbuf)
     return gst_ffmpegaudenc_drain (ffmpegaudenc);
 
+  /* endoder was drained or flushed, and ffmpeg encoder doesn't support
+   * flushing. We need to re-open encoder then */
+  if (ffmpegaudenc->need_reopen) {
+    GST_DEBUG_OBJECT (ffmpegaudenc, "Open encoder again");
+
+    if (!gst_ffmpegaudenc_set_format (encoder,
+            gst_audio_encoder_get_audio_info (encoder))) {
+      GST_ERROR_OBJECT (ffmpegaudenc, "Couldn't re-open encoder");
+      return GST_FLOW_NOT_NEGOTIATED;
+    }
+  }
+
   inbuf = gst_buffer_ref (inbuf);
 
   GST_DEBUG_OBJECT (ffmpegaudenc,
diff --git a/ext/libav/gstavaudenc.h b/ext/libav/gstavaudenc.h
index 23640bb..3c94aef 100644
--- a/ext/libav/gstavaudenc.h
+++ b/ext/libav/gstavaudenc.h
@@ -39,6 +39,7 @@ struct _GstFFMpegAudEnc
   AVCodecContext *context;
   AVCodecContext *refcontext;
   gboolean opened;
+  gboolean need_reopen;
 
   AVFrame *frame;
 
diff --git a/ext/libav/gstavutils.h b/ext/libav/gstavutils.h
index d18b979..7b41b66 100644
--- a/ext/libav/gstavutils.h
+++ b/ext/libav/gstavutils.h
@@ -29,6 +29,26 @@
 
 #include <gst/gst.h>
 
+/* Introduced since ffmpeg version 4.3
+ *
+ * Note: Not all ffmpeg encoders seem to be reusable after flushing/draining.
+ * So if ffmpeg encoder doesn't support it, we should reopen encoding session.
+ *
+ * Before ffmpeg 4.3, avcodec_flush_buffers() was implemented in
+ * libavcodec/decodec.c but it was moved to libavcodec/utils.c and it would be
+ * accepted if encoder supports AV_CODEC_CAP_ENCODER_FLUSH flag.
+ * That implies that avcodec_flush_buffers() wasn't intended to be working
+ * properly for encoders.
+ */
+#ifndef AV_CODEC_CAP_ENCODER_FLUSH
+/*
+ * This encoder can be flushed using avcodec_flush_buffers(). If this flag is
+ * not set, the encoder must be closed and reopened to ensure that no frames
+ * remain pending.
+ */
+#define AV_CODEC_CAP_ENCODER_FLUSH   (1 << 21)
+#endif
+
 /*
  *Get the size of an picture
  */
@@ -79,7 +99,7 @@ gst_ffmpeg_time_gst_to_ff (guint64 time, AVRational base)
   return out;
 }
 
-void 
+void
 gst_ffmpeg_init_pix_fmt_info(void);
 
 int
diff --git a/ext/libav/gstavvidenc.c b/ext/libav/gstavvidenc.c
index 2616e8d..0468d88 100644
--- a/ext/libav/gstavvidenc.c
+++ b/ext/libav/gstavvidenc.c
@@ -243,6 +243,8 @@ gst_ffmpegvidenc_set_format (GstVideoEncoder * encoder,
   GstFFMpegVidEncClass *oclass =
       (GstFFMpegVidEncClass *) G_OBJECT_GET_CLASS (ffmpegenc);
 
+  ffmpegenc->need_reopen = FALSE;
+
   /* close old session */
   if (ffmpegenc->opened) {
     gst_ffmpeg_avcodec_close (ffmpegenc->context);
@@ -622,6 +624,20 @@ gst_ffmpegvidenc_send_frame (GstFFMpegVidEnc * ffmpegenc,
       ffmpegenc->context->ticks_per_frame, ffmpegenc->context->time_base);
 
 send_frame:
+  if (!picture) {
+    GstFFMpegVidEncClass *oclass =
+        (GstFFMpegVidEncClass *) (G_OBJECT_GET_CLASS (ffmpegenc));
+
+    /* If AV_CODEC_CAP_ENCODER_FLUSH wasn't set, we need to re-open
+     * encoder */
+    if (!(oclass->in_plugin->capabilities & AV_CODEC_CAP_ENCODER_FLUSH)) {
+      GST_DEBUG_OBJECT (ffmpegenc, "Encoder needs reopen later");
+
+      /* we will reopen later handle_frame() */
+      ffmpegenc->need_reopen = TRUE;
+    }
+  }
+
   res = avcodec_send_frame (ffmpegenc->context, picture);
 
   if (picture)
@@ -710,6 +726,30 @@ gst_ffmpegvidenc_handle_frame (GstVideoEncoder * encoder,
   GstFlowReturn ret;
   gboolean got_packet;
 
+  /* endoder was drained or flushed, and ffmpeg encoder doesn't support
+   * flushing. We need to re-open encoder then */
+  if (ffmpegenc->need_reopen) {
+    gboolean reopen_ret;
+    GstVideoCodecState *input_state;
+
+    GST_DEBUG_OBJECT (ffmpegenc, "Open encoder again");
+
+    if (!ffmpegenc->input_state) {
+      GST_ERROR_OBJECT (ffmpegenc,
+          "Cannot re-open encoder without input state");
+      return GST_FLOW_NOT_NEGOTIATED;
+    }
+
+    input_state = gst_video_codec_state_ref (ffmpegenc->input_state);
+    reopen_ret = gst_ffmpegvidenc_set_format (encoder, input_state);
+    gst_video_codec_state_unref (input_state);
+
+    if (!reopen_ret) {
+      GST_ERROR_OBJECT (ffmpegenc, "Couldn't re-open encoder");
+      return GST_FLOW_NOT_NEGOTIATED;
+    }
+  }
+
   ret = gst_ffmpegvidenc_send_frame (ffmpegenc, frame);
 
   if (ret != GST_FLOW_OK)
@@ -852,6 +892,9 @@ gst_ffmpegvidenc_start (GstVideoEncoder * encoder)
   GstFFMpegVidEncClass *oclass =
       (GstFFMpegVidEncClass *) G_OBJECT_GET_CLASS (ffmpegenc);
 
+  ffmpegenc->opened = FALSE;
+  ffmpegenc->need_reopen = FALSE;
+
   /* close old session */
   gst_ffmpeg_avcodec_close (ffmpegenc->context);
   if (avcodec_get_context_defaults3 (ffmpegenc->context, oclass->in_plugin) < 0) {
@@ -872,6 +915,7 @@ gst_ffmpegvidenc_stop (GstVideoEncoder * encoder)
   gst_ffmpegvidenc_flush_buffers (ffmpegenc, FALSE);
   gst_ffmpeg_avcodec_close (ffmpegenc->context);
   ffmpegenc->opened = FALSE;
+  ffmpegenc->need_reopen = FALSE;
 
   if (ffmpegenc->input_state) {
     gst_video_codec_state_unref (ffmpegenc->input_state);
diff --git a/ext/libav/gstavvidenc.h b/ext/libav/gstavvidenc.h
index d81d72b..2d0b7c8 100644
--- a/ext/libav/gstavvidenc.h
+++ b/ext/libav/gstavvidenc.h
@@ -41,6 +41,7 @@ struct _GstFFMpegVidEnc
   AVCodecContext *context;
   AVFrame *picture;
   gboolean opened;
+  gboolean need_reopen;
   gboolean discont;
   guint pass;
   gfloat quantizer;