avviddec: Fix huge leak caused by circular reference

AVBufferRef -> GstFFMpegVideoDecVideoFrame -> GstVideoCodecFrame -> AVBufferRef

Instead of holding additional ref there, set read-only which would not be
reused by ff_reget_buffer()

Fixes: https://gitlab.freedesktop.org/gstreamer/gst-libav/issues/63

1 files changed, 5 insertions, 11 deletions

diff --git a/ext/libav/gstavviddec.c b/ext/libav/gstavviddec.c
index 58979d8..b0f55f2 100644
--- a/ext/libav/gstavviddec.c
+++ b/ext/libav/gstavviddec.c
@@ -730,12 +730,6 @@ gst_ffmpegviddec_can_direct_render (GstFFMpegVidDec * ffmpegdec)
       AV_CODEC_CAP_DR1);
 }
 
-static void
-gst_ffmpegviddec_avbuffer_unref (AVBufferRef * avbuffer)
-{
-  av_buffer_unref (&avbuffer);
-}
-
 /* called when ffmpeg wants us to allocate a buffer to write the decoded frame
  * into. We try to give it memory from our pool */
 static int
@@ -747,6 +741,7 @@ gst_ffmpegviddec_get_buffer2 (AVCodecContext * context, AVFrame * picture,
   GstFFMpegVidDec *ffmpegdec;
   guint c;
   GstFlowReturn ret;
+  int create_buffer_flags = 0;
 
   ffmpegdec = (GstFFMpegVidDec *) context->opaque;
 
@@ -828,17 +823,16 @@ gst_ffmpegviddec_get_buffer2 (AVCodecContext * context, AVFrame * picture,
         picture->data[c]);
   }
 
-  picture->buf[0] = av_buffer_create (NULL, 0, dummy_free_buffer, dframe, 0);
   if ((flags & AV_GET_BUFFER_FLAG_REF) == AV_GET_BUFFER_FLAG_REF) {
     /* decoder might reuse this AVFrame and it would result to no more
      * get_buffer() call if the AVFrame's AVBuffer is writable
      * (meaning that the refcount of AVBuffer == 1).
-     * To enforce get_buffer() for the every output frame, hold another ref here
+     * To enforce get_buffer() for the every output frame, set read-only flag here
      */
-    gst_video_codec_frame_set_user_data (frame,
-        av_buffer_ref (picture->buf[0]),
-        (GDestroyNotify) gst_ffmpegviddec_avbuffer_unref);
+    create_buffer_flags = AV_BUFFER_FLAG_READONLY;
   }
+  picture->buf[0] = av_buffer_create (NULL,
+      0, dummy_free_buffer, dframe, create_buffer_flags);
 
   GST_LOG_OBJECT (ffmpegdec, "returned frame %p", dframe->buffer);