mpeg2enc: Only allow 1 pending frame for encoding

Having an unlimited input queue is very bad if the
encoder can't run at real-time. Eventually it will
consume all RAM. I don't really see any reason to
have more than 1 outstanding encoded frame, so
remove the queue and limit things to 1 pending frame.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2499>

3 files changed, 25 insertions, 16 deletions

diff --git a/ext/mpeg2enc/gstmpeg2enc.cc b/ext/mpeg2enc/gstmpeg2enc.cc
index 1eaa5b391..e209248ae 100644
--- a/ext/mpeg2enc/gstmpeg2enc.cc
+++ b/ext/mpeg2enc/gstmpeg2enc.cc
@@ -163,7 +163,6 @@ gst_mpeg2enc_finalize (GObject * object)
 
   delete enc->options;
 
-  g_queue_free (enc->frames);
   g_mutex_clear (&enc->tlock);
   g_cond_clear (&enc->cond);
 
@@ -178,7 +177,6 @@ gst_mpeg2enc_init (GstMpeg2enc * enc)
 
   g_mutex_init (&enc->tlock);
   g_cond_init (&enc->cond);
-  enc->frames = g_queue_new ();
   enc->started = FALSE;
 
   gst_pad_set_activatemode_function (GST_VIDEO_ENCODER_SRC_PAD (enc),
@@ -216,13 +214,14 @@ gst_mpeg2enc_src_activate_mode (GstPad * pad, GstObject * parent,
 static void
 gst_mpeg2enc_reset (GstMpeg2enc * enc)
 {
-  GstVideoCodecFrame *frame;
-
   enc->eos = FALSE;
   enc->srcresult = GST_FLOW_OK;
 
   /* in case of error'ed ending */
-  while ((frame = (GstVideoCodecFrame *) g_queue_pop_head (enc->frames)));
+  if (enc->pending_frame) {
+    gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (enc), enc->pending_frame);
+    enc->pending_frame = NULL;
+  }
 
   if (enc->encoder) {
     delete enc->encoder;
@@ -529,7 +528,7 @@ gst_mpeg2enc_sink_event (GstVideoEncoder * video_encoder, GstEvent * event)
        * though this is no guarantee as to when the encoder is done with it */
       if (GST_EVENT_IS_SERIALIZED (event)) {
         GST_MPEG2ENC_MUTEX_LOCK (enc);
-        while (g_queue_get_length (enc->frames))
+        while (enc->pending_frame != NULL)
           GST_MPEG2ENC_WAIT (enc);
         GST_MPEG2ENC_MUTEX_UNLOCK (enc);
       }
@@ -678,21 +677,31 @@ gst_mpeg2enc_handle_frame (GstVideoEncoder *video_encoder, GstVideoCodecFrame *f
     goto ignore;
   GST_DEBUG_OBJECT (video_encoder, "handle_frame: flow OK");
 
-  g_queue_push_tail (enc->frames, frame);
+  /* If the encoder is busy with a previous frame still, wait
+   * for it to be done */
+  if (enc->pending_frame != NULL) {
+    do {
+      GST_VIDEO_ENCODER_STREAM_UNLOCK (enc);
+      GST_MPEG2ENC_WAIT (enc);
+      GST_VIDEO_ENCODER_STREAM_LOCK (enc);
+
+      /* Re-check the srcresult, since we waited */
+      if (G_UNLIKELY (enc->srcresult != GST_FLOW_OK))
+        goto ignore;
+    } while (enc->pending_frame != NULL);
+  }
 
   /* frame will be released by task */
+  enc->pending_frame = frame;
 
-  if (g_queue_get_length (enc->frames) > 0 && !enc->started) {
+  if (!enc->started) {
     GST_DEBUG_OBJECT (video_encoder, "handle_frame: START task");
     gst_pad_start_task (video_encoder->srcpad, (GstTaskFunction) gst_mpeg2enc_loop, enc, NULL);
     enc->started = TRUE;
-
   }
 
   /* things look good, now inform the encoding task that a frame is ready */
-  if (enc->started)
-    GST_MPEG2ENC_SIGNAL (enc);
-
+  GST_MPEG2ENC_SIGNAL (enc);
   GST_MPEG2ENC_MUTEX_UNLOCK (enc);
 
   return GST_FLOW_OK;
diff --git a/ext/mpeg2enc/gstmpeg2enc.hh b/ext/mpeg2enc/gstmpeg2enc.hh
index fcc3fb933..369276b55 100644
--- a/ext/mpeg2enc/gstmpeg2enc.hh
+++ b/ext/mpeg2enc/gstmpeg2enc.hh
@@ -85,13 +85,11 @@ typedef struct _GstMpeg2enc {
   gboolean eos;
   /* flowreturn obtained by encoding task */
   GstFlowReturn srcresult;
-  /* frames for input */
-  GQueue *frames;
 
   gboolean started;
 
   GstVideoCodecState *input_state;
-
+  GstVideoCodecFrame *pending_frame;
 } GstMpeg2enc;
 
 typedef struct _GstMpeg2encClass {
diff --git a/ext/mpeg2enc/gstmpeg2encpicturereader.cc b/ext/mpeg2enc/gstmpeg2encpicturereader.cc
index 3641b613a..b0c5484d9 100644
--- a/ext/mpeg2enc/gstmpeg2encpicturereader.cc
+++ b/ext/mpeg2enc/gstmpeg2encpicturereader.cc
@@ -135,7 +135,7 @@ bool
   GST_MPEG2ENC_MUTEX_LOCK (enc);
 
   /* hang around until the element provides us with a buffer */
-  while (!(inframe = (GstVideoCodecFrame *)g_queue_pop_head (enc->frames))) {
+  while (enc->pending_frame == NULL) {
     if (enc->eos) {
       GST_MPEG2ENC_MUTEX_UNLOCK (enc);
       /* inform the mpeg encoding loop that it can give up */
@@ -144,7 +144,9 @@ bool
     GST_MPEG2ENC_WAIT (enc);
   }
 
+  inframe = enc->pending_frame;
   gst_video_frame_map (&vframe, &enc->input_state->info, inframe->input_buffer, GST_MAP_READ);
+  enc->pending_frame = NULL;
 
   frame = GST_VIDEO_FRAME_COMP_DATA (&vframe, 0);
   s = GST_VIDEO_FRAME_COMP_STRIDE (&vframe, 0);