msdkdec: avoid infinite loop

It is possible MFXVideoDECODE_DecodeFrameAsync returns MFX_ERR_INCOMPATIBLE_VIDEO_PARAM
and this error can't be recovered by retrying MFXVideoDECODE_DecodeFrameAsync
in some cases, so we need to limit the number of retries to avoid infinite loop.

This fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/909

1 files changed, 5 insertions, 2 deletions

diff --git a/sys/msdk/gstmsdkdec.c b/sys/msdk/gstmsdkdec.c
index b7c9b4c9d..53faaf937 100644
--- a/sys/msdk/gstmsdkdec.c
+++ b/sys/msdk/gstmsdkdec.c
@@ -853,7 +853,7 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
   mfxSession session;
   mfxStatus status;
   GstMapInfo map_info;
-  guint i;
+  guint i, retry_err_incompatible = 0;
   gsize data_size;
   gboolean hard_reset = FALSE;
 
@@ -1000,7 +1000,8 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
 
     /* media-sdk requires complete reset since the surface is inadaquate to
      * do further decoding */
-    if (status == MFX_ERR_INCOMPATIBLE_VIDEO_PARAM) {
+    if (status == MFX_ERR_INCOMPATIBLE_VIDEO_PARAM &&
+        retry_err_incompatible++ < 1) {
       /* MFX_ERR_INCOMPATIBLE_VIDEO_PARAM means the current mfx surface is not
        * suitable for the current frame, call MFXVideoDECODE_DecodeHeader to get
        * the current frame size then do memory re-allocation, otherwise
@@ -1021,6 +1022,8 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
       continue;
     }
 
+    retry_err_incompatible = 0;
+
     if (G_LIKELY (status == MFX_ERR_NONE)
         || (status == MFX_WRN_VIDEO_PARAM_CHANGED)) {
       thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;