msdk: dec: Add dynamic-configuration change support

The patch adds a serios of changes to support dynamic resolution
change and efficient utilization of resources.
Major changes:

-- Use MSDK's apis to retrieve the headers instead of only relying
on upsteram notification. For eg: avc decoder requires SEI header
information for dpb count calculation which we don't get from caps.

-- For all codecs other than VP9, we force the reset of decoder
if resoultion changes to fit with gstreamer flow. VP9 enfource
the hard reset only if the new resolution is bigger.

-- delay the src caps setting till msdk api's invokation in
handle_frame to avoid caching multiple configuration values

-- ensure pool negotiation is based on decoder's allocation_caps.

--dynamic resoluttion change use an explicit allocation_query
to reclaim the buffers before closing the decoder (thanks to v4l2dec)

--In case if we don't get upstream notification of res change (for eg,
this can can happen for vp9 frames with ivfheader where ivfparse
is not able to notify the dynamic changes), we handle the the case
based on MFX_ERR_INCOMPATIBLE_VIDEO_PARAM which is the return value
of MFXVideoDECODE_DecodeFrameAsync

-- calculate the minimum surfaces to be preallocated based on
msdk suggestion, downstream requirement, async depth and scratch surface
count for smooth display.

https://bugzilla.gnome.org/show_bug.cgi?id=796566

3 files changed, 334 insertions, 96 deletions

diff --git a/sys/msdk/gstmsdkdec.c b/sys/msdk/gstmsdkdec.c
index a0d8baa71..fb8470537 100644
--- a/sys/msdk/gstmsdkdec.c
+++ b/sys/msdk/gstmsdkdec.c
@@ -69,6 +69,8 @@ enum
 #define PROP_ASYNC_DEPTH_DEFAULT         1
 #define PROP_OUTPUT_ORDER_DEFAULT        GST_MSDKDEC_OUTPUT_ORDER_DISPLAY
 
+#define IS_ALIGNED(i, n) (((i) & ((n)-1)) == 0)
+
 #define gst_msdkdec_parent_class parent_class
 G_DEFINE_TYPE (GstMsdkDec, gst_msdkdec, GST_TYPE_VIDEO_DECODER);
 
@@ -80,7 +82,9 @@ typedef struct _MsdkSurface
   GstVideoFrame copy;
 } MsdkSurface;
 
+static gboolean gst_msdkdec_drain (GstVideoDecoder * decoder);
 static gboolean gst_msdkdec_flush (GstVideoDecoder * decoder);
+static gboolean gst_msdkdec_negotiate (GstMsdkDec * thiz, gboolean hard_reset);
 
 static GstFlowReturn
 allocate_output_buffer (GstMsdkDec * thiz, GstBuffer ** buffer)
@@ -146,7 +150,7 @@ get_surface (GstMsdkDec * thiz, GstBuffer * buffer)
       return NULL;
     }
 
-    if (!gst_video_frame_map (&i->copy, &thiz->output_info, buffer,
+    if (!gst_video_frame_map (&i->copy, &thiz->non_msdk_pool_info, buffer,
             GST_MAP_WRITE))
       goto failed_unref_buffer;
 
@@ -157,7 +161,7 @@ get_surface (GstMsdkDec * thiz, GstBuffer * buffer)
     i->surface = gst_msdk_get_surface_from_buffer (buffer);
     i->buf = buffer;
 
-    if (!gst_video_frame_map (&i->data, &thiz->pool_info, buffer,
+    if (!gst_video_frame_map (&i->data, &thiz->output_info, buffer,
             GST_MAP_READWRITE))
       goto failed_unref_buffer2;
   }
@@ -179,7 +183,7 @@ failed_unref_buffer:
 }
 
 static void
-gst_msdkdec_close_decoder (GstMsdkDec * thiz)
+gst_msdkdec_close_decoder (GstMsdkDec * thiz, gboolean reset_param)
 {
   mfxStatus status;
 
@@ -201,7 +205,9 @@ gst_msdkdec_close_decoder (GstMsdkDec * thiz)
   g_array_set_size (thiz->tasks, 0);
   g_ptr_array_set_size (thiz->extra_params, 0);
 
-  memset (&thiz->param, 0, sizeof (thiz->param));
+  if (reset_param)
+    memset (&thiz->param, 0, sizeof (thiz->param));
+
   thiz->initialized = FALSE;
   gst_adapter_clear (thiz->adapter);
 }
@@ -224,7 +230,6 @@ gst_msdkdec_set_context (GstElement * element, GstContext * context)
 static gboolean
 gst_msdkdec_init_decoder (GstMsdkDec * thiz)
 {
-  GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
   GstVideoInfo *info;
   mfxSession session;
   mfxStatus status;
@@ -257,34 +262,48 @@ gst_msdkdec_init_decoder (GstMsdkDec * thiz)
       thiz->use_video_memory ? "video" : "system");
 
   thiz->param.AsyncDepth = thiz->async_depth;
-  thiz->param.mfx.FrameInfo.Width = GST_ROUND_UP_32 (info->width);
-  thiz->param.mfx.FrameInfo.Height = GST_ROUND_UP_32 (info->height);
-  thiz->param.mfx.FrameInfo.CropW = info->width;
-  thiz->param.mfx.FrameInfo.CropH = info->height;
+  /* This is a deprecated attribute in msdk-2017 version, but some
+   * customers still using this for low-latency streaming of non-b-frame
+   * encoded streams */
+  thiz->param.mfx.DecodedOrder = thiz->output_order;
 
+  /* We expect msdk to fill the width and height values */
+  g_return_val_if_fail (thiz->param.mfx.FrameInfo.Width
+      && thiz->param.mfx.FrameInfo.Height, FALSE);
+
+  /* Force 32 bit rounding to avoid messing up of memory alignment when
+   * dealing with different allocators */
+  /* Fixme: msdk sometimes only requires 16 bit rounding, optimization possible */
+  thiz->param.mfx.FrameInfo.Width =
+      GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Width);
+  thiz->param.mfx.FrameInfo.Height =
+      GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Height);
   /* Set framerate only if provided.
-   * If not, framerate will be assumed inside the driver */
-  if (info->fps_n > 0 && info->fps_d > 0) {
+   * If not, framerate will be assumed inside the driver.
+   * Also we respect the upstream provided fps values */
+  if (info->fps_n > 0 && info->fps_d > 0
+      && info->fps_n != thiz->param.mfx.FrameInfo.FrameRateExtN
+      && info->fps_d != thiz->param.mfx.FrameInfo.FrameRateExtD) {
     thiz->param.mfx.FrameInfo.FrameRateExtN = info->fps_n;
     thiz->param.mfx.FrameInfo.FrameRateExtD = info->fps_d;
   }
-  thiz->param.mfx.FrameInfo.AspectRatioW = info->par_n;
-  thiz->param.mfx.FrameInfo.AspectRatioH = info->par_d;
-  thiz->param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
-  thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12;
-  thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420;
-  thiz->param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
 
-  /* This is a deprecated attribute in msdk-2017 version, but some
-   * customers still using this for low-latency streaming of non-b-frame
-   * encoded streams */
-  thiz->param.mfx.DecodedOrder = thiz->output_order;
-  /* allow subclass configure further */
-  if (klass->configure) {
-    if (!klass->configure (thiz))
-      goto failed;
+  if (info->par_n && info->par_d && !thiz->param.mfx.FrameInfo.AspectRatioW
+      && !thiz->param.mfx.FrameInfo.AspectRatioH) {
+    thiz->param.mfx.FrameInfo.AspectRatioW = info->par_n;
+    thiz->param.mfx.FrameInfo.AspectRatioH = info->par_d;
   }
 
+  thiz->param.mfx.FrameInfo.PicStruct =
+      thiz->param.mfx.FrameInfo.PicStruct ? thiz->param.mfx.
+      FrameInfo.PicStruct : MFX_PICSTRUCT_PROGRESSIVE;
+  thiz->param.mfx.FrameInfo.FourCC =
+      thiz->param.mfx.FrameInfo.FourCC ? thiz->param.mfx.
+      FrameInfo.FourCC : MFX_FOURCC_NV12;
+  thiz->param.mfx.FrameInfo.ChromaFormat =
+      thiz->param.mfx.FrameInfo.ChromaFormat ? thiz->param.mfx.
+      FrameInfo.ChromaFormat : MFX_CHROMAFORMAT_YUV420;
+
   thiz->param.NumExtParam = thiz->extra_params->len;
   thiz->param.ExtParam = (mfxExtBuffer **) thiz->extra_params->pdata;
 
@@ -316,6 +335,14 @@ gst_msdkdec_init_decoder (GstMsdkDec * thiz)
     goto failed;
   }
 
+  /* account the downstream requirement */
+  if (G_LIKELY (thiz->min_prealloc_buffers))
+    request.NumFrameSuggested += thiz->min_prealloc_buffers;
+  else
+    GST_WARNING_OBJECT (thiz,
+        "Allocating resources without considering the downstream requirement"
+        "or extra scratch surface count");
+
   if (thiz->use_video_memory) {
     gint shared_async_depth;
 
@@ -329,6 +356,10 @@ gst_msdkdec_init_decoder (GstMsdkDec * thiz)
     gst_msdk_frame_alloc (thiz->context, &request, &thiz->alloc_resp);
   }
 
+  /* update the prealloc_buffer count which will be used later
+   * as GstBufferPool min_buffers */
+  thiz->min_prealloc_buffers = request.NumFrameSuggested;
+
   GST_DEBUG_OBJECT (thiz, "Required %d surfaces (%d suggested)",
       request.NumFrameMin, request.NumFrameSuggested);
 
@@ -413,32 +444,72 @@ done:
 }
 
 static gboolean
-gst_msdkdec_set_src_caps (GstMsdkDec * thiz)
+gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation)
 {
   GstVideoCodecState *output_state;
+  GstVideoInfo *vinfo;
   GstVideoAlignment align;
+  GstCaps *allocation_caps = NULL;
   guint width, height;
-
-  width = GST_VIDEO_INFO_WIDTH (&thiz->input_state->info);
-  height = GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info);
-
+  const gchar *format_str;
+
+  /* use display width and display height in output state which
+   * will be using for caps negotiation */
+  width =
+      thiz->param.mfx.FrameInfo.CropW ? thiz->param.mfx.
+      FrameInfo.CropW : GST_VIDEO_INFO_WIDTH (&thiz->input_state->info);
+  height =
+      thiz->param.mfx.FrameInfo.CropH ? thiz->param.mfx.
+      FrameInfo.CropH : GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info);
   output_state =
       gst_video_decoder_set_output_state (GST_VIDEO_DECODER (thiz),
       GST_VIDEO_FORMAT_NV12, width, height, thiz->input_state);
+  if (!output_state)
+    return FALSE;
 
-  gst_msdk_set_video_alignment (&output_state->info, &align);
-  gst_video_info_align (&output_state->info, &align);
-  thiz->output_info = output_state->info;
-  if (output_state->caps)
-    gst_caps_unref (output_state->caps);
-  output_state->caps = gst_video_info_to_caps (&output_state->info);
-
+  /* Ensure output_state->caps and info has same width and height
+   * Also mandate the 32 bit alignment */
+  vinfo = &output_state->info;
+  gst_msdk_set_video_alignment (vinfo, &align);
+  gst_video_info_align (vinfo, &align);
+  output_state->caps = gst_video_info_to_caps (vinfo);
   if (srcpad_can_dmabuf (thiz))
     gst_caps_set_features (output_state->caps, 0,
         gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_DMABUF, NULL));
+  thiz->output_info = output_state->info;
 
-  gst_video_codec_state_unref (output_state);
+  if (need_allocation) {
+    /* Find allocation width and height */
+    width =
+        GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Width ? thiz->param.mfx.
+        FrameInfo.Width : GST_VIDEO_INFO_WIDTH (&output_state->info));
+    height =
+        GST_ROUND_UP_32 (thiz->param.mfx.FrameInfo.Height ? thiz->param.mfx.
+        FrameInfo.Height : GST_VIDEO_INFO_HEIGHT (&output_state->info));
+
+    /* set allocation width and height in allocation_caps
+     * which may or may not be similar to the output_state caps */
+    allocation_caps = gst_caps_copy (output_state->caps);
+    format_str =
+        gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&thiz->output_info));
+    gst_caps_set_simple (allocation_caps, "width", G_TYPE_INT, width, "height",
+        G_TYPE_INT, height, "format", G_TYPE_STRING, format_str, NULL);
+    GST_INFO_OBJECT (thiz, "new alloc caps = %" GST_PTR_FORMAT,
+        allocation_caps);
+    gst_caps_replace (&thiz->allocation_caps, allocation_caps);
+  } else {
+    /* We keep the allocation parameters as it is to avoid pool renegotiation.
+     * For codecs like VP9, dynamic resolution change doesn't requires allocation
+     * reset if the new video frame resolution is lower than the
+     * already configured one */
+    allocation_caps = gst_caps_copy (thiz->allocation_caps);
+  }
 
+  gst_caps_replace (&output_state->allocation_caps, allocation_caps);
+  if (allocation_caps)
+    gst_caps_unref (allocation_caps);
+
+  gst_video_codec_state_unref (output_state);
   return TRUE;
 }
 
@@ -595,9 +666,9 @@ gst_msdkdec_stop (GstVideoDecoder * decoder)
     thiz->pool = NULL;
   }
   gst_video_info_init (&thiz->output_info);
-  gst_video_info_init (&thiz->pool_info);
+  gst_video_info_init (&thiz->non_msdk_pool_info);
 
-  gst_msdkdec_close_decoder (thiz);
+  gst_msdkdec_close_decoder (thiz, TRUE);
   return TRUE;
 }
 
@@ -606,12 +677,21 @@ gst_msdkdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
 {
   GstMsdkDec *thiz = GST_MSDKDEC (decoder);
 
-  if (thiz->input_state)
+  if (thiz->input_state) {
+    /* mark for re-negotiation if display resolution changes */
+    if ((GST_VIDEO_INFO_WIDTH (&thiz->input_state->info) !=
+            GST_VIDEO_INFO_WIDTH (&state->info)) ||
+        GST_VIDEO_INFO_HEIGHT (&thiz->input_state->info) !=
+        GST_VIDEO_INFO_HEIGHT (&state->info))
+      thiz->do_renego = TRUE;
     gst_video_codec_state_unref (thiz->input_state);
+  }
   thiz->input_state = gst_video_codec_state_ref (state);
 
-  if (!gst_msdkdec_set_src_caps (thiz))
-    return FALSE;
+  /* we don't set output state here to avoid caching of mismatched
+   * video information if there is dynamic resolution change in the stream.
+   * All negotiation code is consolidated in gst_msdkdec_negotiate() and
+   * this will be invoked from handle_frame() */
 
   gst_msdkdec_set_latency (thiz);
   return TRUE;
@@ -629,12 +709,89 @@ release_msdk_surfaces (GstMsdkDec * thiz)
   }
 }
 
+/* This will get invoked in the following situations:
+ * 1: begining of the stream, which requires initialization (== complete reset)
+ * 2: upstream notified a resolution change and set do_renego to TRUE.
+ *    new resoulution may or may not requires full reset
+ * 3: upstream failed to notify the resoulution change but
+ *    msdk detected the change (eg: vp9 stream in ivf elementary form
+ *     with varying resolution frames).
+ *
+ * for any input configuration change, we deal with notification
+ * from upstream and also use msdk apis to handle the parameter initialization
+ * efficiently
+ */
+static gboolean
+gst_msdkdec_negotiate (GstMsdkDec * thiz, gboolean hard_reset)
+{
+  GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
+  GST_DEBUG_OBJECT (thiz,
+      "Start Negotiating caps, pool and Init the msdk decdoer subsystem");
+
+  if (hard_reset) {
+    /* Retrieve any pending frames and push them downstream */
+    if (gst_msdkdec_drain (GST_VIDEO_DECODER (thiz)) != GST_FLOW_OK)
+      goto error_drain;
+
+    /* This will initiate the allocation query which will help to flush
+     * all the pending buffers in the pipeline so that we can stop
+     * the active bufferpool and safely invoke gst_msdk_frame_free() */
+    if (thiz->initialized) {
+      GstCaps *caps = gst_pad_get_current_caps (decoder->srcpad);
+      GstQuery *query = NULL;
+      if (caps) {
+        query = gst_query_new_allocation (caps, FALSE);
+        gst_pad_peer_query (decoder->srcpad, query);
+        gst_query_unref (query);
+        gst_caps_unref (caps);
+      }
+    }
+
+    /* De-initialize the decoder if it is already active */
+    /* Not resetting the mfxVideoParam since it already
+     * possessing the required parameters for new session decode */
+    gst_msdkdec_close_decoder (thiz, FALSE);
+
+    /* request for pool renegotiation by setting do_realloc */
+    thiz->do_realloc = TRUE;
+  }
+
+  /* At this point all pending frames(if there is any) are pushed downsteram
+   * and we are ready to negotiate the output caps */
+  if (!gst_msdkdec_set_src_caps (thiz, hard_reset))
+    return FALSE;
+
+  /* this will initiate the allocation query, we create the
+   * bufferpool in decide_allocation inorder to account
+   * the downstream min_buffer requirement
+   * Required initializations for MediaSDK operations
+   * will all be inited from decide_allocation after considering
+   * some of the downstream requirements */
+  if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (thiz)))
+    goto error_negotiate;
+
+  thiz->do_renego = FALSE;
+  thiz->do_realloc = FALSE;
+
+  return TRUE;
+
+error_drain:
+  GST_ERROR_OBJECT (thiz, "Failed to Drain the queued decoded frames");
+  return FALSE;
+
+error_negotiate:
+  GST_ERROR_OBJECT (thiz, "Failed to renegotiation");
+  return FALSE;
+}
+
 static GstFlowReturn
 gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
 {
   GstMsdkDec *thiz = GST_MSDKDEC (decoder);
+  GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz);
   GstFlowReturn flow;
   GstBuffer *buffer;
+  GstVideoInfo alloc_info;
   MsdkDecTask *task = NULL;
   mfxBitstream bitstream;
   MsdkSurface *surface = NULL;
@@ -643,9 +800,7 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
   GstMapInfo map_info;
   guint i;
   gsize data_size;
-
-  if (!thiz->initialized)
-    gst_video_decoder_negotiate (decoder);
+  gboolean hard_reset = FALSE;
 
   if (!gst_buffer_map (frame->input_buffer, &map_info, GST_MAP_READ))
     return GST_FLOW_ERROR;
@@ -673,6 +828,47 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
       bitstream.DataLength, bitstream.DataOffset, bitstream.MaxLength);
 
   session = gst_msdk_context_get_session (thiz->context);
+
+  if (!thiz->initialized || thiz->do_renego) {
+
+    /* configure the subclss inorder to fill the CodecID field of mfxVideoParam
+     * and also to load the PluginID for some of the codecs which is mandatory
+     * to invoke the MFXVideoDECODE_DecodeHeader API */
+    if (!klass->configure || !klass->configure (thiz)) {
+      flow = GST_FLOW_OK;
+      goto error;
+    }
+
+    /* gstreamer caps will not bring all the necessary parameters
+     * required for optimal decode configuration. For eg: the required numbers
+     * of surfaces to be allocated can be calculated based on H264 SEI header
+     * and this information can't be retrieved from the negotiated caps.
+     * So instead of introducing the codecparser dependency to parse the headers
+     * inside msdk plugin, we simply use the mfx apis to extract header information */
+    status = MFXVideoDECODE_DecodeHeader (session, &bitstream, &thiz->param);
+    if (status == MFX_ERR_MORE_DATA) {
+      flow = GST_FLOW_OK;
+      goto done;
+    }
+
+    if (thiz->initialized && thiz->allocation_caps)
+      gst_video_info_from_caps (&alloc_info, thiz->allocation_caps);
+
+    /* Check whether we need complete reset for dynamic resolution change */
+    if (!thiz->initialized || (thiz->initialized &&
+            (thiz->param.mfx.FrameInfo.Width >
+                GST_VIDEO_INFO_WIDTH (&alloc_info)
+                || thiz->param.mfx.FrameInfo.Height >
+                GST_VIDEO_INFO_HEIGHT (&alloc_info))))
+      hard_reset = TRUE;
+
+    /* if subclass requested for the force reset */
+    if (thiz->force_reset_on_res_change)
+      hard_reset = TRUE;
+
+    gst_msdkdec_negotiate (thiz, hard_reset);
+  }
+
   for (;;) {
     task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
     flow = gst_msdkdec_finish_task (thiz, task);
@@ -707,7 +903,20 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
     status =
         MFXVideoDECODE_DecodeFrameAsync (session, &bitstream, surface->surface,
         &task->surface, &task->sync_point);
-    if (G_LIKELY (status == MFX_ERR_NONE)) {
+
+    /* media-sdk requires complete reset since the surface is inadaquate to
+     * do further decoding */
+    if (status == MFX_ERR_INCOMPATIBLE_VIDEO_PARAM) {
+      /* Requires memory re-allocation ,initiate hard reset */
+      if (!gst_msdkdec_negotiate (thiz, TRUE))
+        goto error;
+      status =
+          MFXVideoDECODE_DecodeFrameAsync (session, &bitstream,
+          surface->surface, &task->surface, &task->sync_point);
+    }
+
+    if (G_LIKELY (status == MFX_ERR_NONE)
+        || (status == MFX_WRN_VIDEO_PARAM_CHANGED)) {
       thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
 
       if (surface->surface->Data.Locked > 0 || !thiz->use_video_memory)
@@ -747,6 +956,7 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
     flow = GST_FLOW_OK;
   }
 
+done:
   if (surface)
     free_surface (thiz, surface);
 
@@ -760,45 +970,52 @@ error:
   return flow;
 }
 
+
 static GstBufferPool *
-gst_msdkdec_create_buffer_pool (GstMsdkDec * thiz, GstCaps * caps,
+gst_msdkdec_create_buffer_pool (GstMsdkDec * thiz, GstVideoInfo * info,
     guint num_buffers)
 {
   GstBufferPool *pool = NULL;
   GstStructure *config;
   GstAllocator *allocator = NULL;
-  GstVideoInfo info;
   GstVideoAlignment align;
+  GstCaps *caps = NULL;
   GstAllocationParams params = { 0, 31, 0, 0, };
   mfxFrameAllocResponse *alloc_resp = NULL;
 
+  g_return_val_if_fail (info, NULL);
+  g_return_val_if_fail (GST_VIDEO_INFO_WIDTH (info)
+      && GST_VIDEO_INFO_HEIGHT (info), NULL);
+
   alloc_resp = &thiz->alloc_resp;
 
   pool = gst_msdk_buffer_pool_new (thiz->context, alloc_resp);
   if (!pool)
     goto error_no_pool;
 
-  if (!gst_video_info_from_caps (&info, caps)) {
-    GST_INFO_OBJECT (thiz, "failed to get video info");
-    return FALSE;
+  if (G_UNLIKELY (!IS_ALIGNED (GST_VIDEO_INFO_WIDTH (info), 32)
+          || !IS_ALIGNED (GST_VIDEO_INFO_HEIGHT (info), 32))) {
+    gst_msdk_set_video_alignment (info, &align);
+    gst_video_info_align (info, &align);
   }
 
-  gst_msdk_set_video_alignment (&info, &align);
-  gst_video_info_align (&info, &align);
+  caps = gst_video_info_to_caps (info);
 
+  /* allocators should use the same width/height/stride/height_alignment of
+   * negotiated output caps which is what we configure in msdk_allocator */
   if (thiz->use_dmabuf)
-    allocator =
-        gst_msdk_dmabuf_allocator_new (thiz->context, &info, alloc_resp);
+    allocator = gst_msdk_dmabuf_allocator_new (thiz->context, info, alloc_resp);
   else if (thiz->use_video_memory)
-    allocator = gst_msdk_video_allocator_new (thiz->context, &info, alloc_resp);
+    allocator = gst_msdk_video_allocator_new (thiz->context, info, alloc_resp);
   else
-    allocator = gst_msdk_system_allocator_new (&info);
+    allocator = gst_msdk_system_allocator_new (info);
 
   if (!allocator)
     goto error_no_allocator;
 
   config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));
-  gst_buffer_pool_config_set_params (config, caps, info.size, num_buffers, 0);
+  gst_buffer_pool_config_set_params (config, caps,
+      GST_VIDEO_INFO_SIZE (info), num_buffers, 0);
   gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
   gst_buffer_pool_config_add_option (config,
       GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
@@ -841,11 +1058,9 @@ static gboolean
 gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
 {
   GstMsdkDec *thiz = GST_MSDKDEC (decoder);
-  GstVideoInfo info_from_caps, info_aligned;
-  GstVideoAlignment alignment;
   GstBufferPool *pool = NULL;
   GstStructure *pool_config = NULL;
-  GstCaps *pool_caps;
+  GstCaps *pool_caps /*, *negotiated_caps */ ;
   guint size, min_buffers, max_buffers;
 
   if (!GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation (decoder,
@@ -866,11 +1081,39 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
   if (max_buffers)
     max_buffers += thiz->async_depth;
 
+  /* increase the min_buffers by 1 for smooth display in render pipeline */
+  min_buffers += 1;
+
+  /* this will get updated with msdk requirement */
+  thiz->min_prealloc_buffers = min_buffers;
+
   if (_gst_caps_has_feature (pool_caps, GST_CAPS_FEATURE_MEMORY_DMABUF)) {
     GST_INFO_OBJECT (decoder, "This MSDK decoder uses DMABuf memory");
     thiz->use_video_memory = thiz->use_dmabuf = TRUE;
   }
 
+  /* Initialize MSDK decoder before new bufferpool tries to alloc each buffer,
+   * which requires information of frame allocation.
+   * No effect if already initialized.
+   */
+  if (!gst_msdkdec_init_decoder (thiz))
+    return FALSE;
+
+  /* get the updated min_buffers which account the msdk requirement too */
+  min_buffers = thiz->min_prealloc_buffers;
+
+  /* Decoder always use its own pool. So we create a pool if msdk apis
+   * previously requested for allocation (do_realloc = TRUE) */
+  if (thiz->do_realloc || !thiz->pool) {
+    if (thiz->pool)
+      gst_object_replace ((GstObject **) & thiz->pool, NULL);
+    GST_INFO_OBJECT (decoder, "create new MSDK bufferpool");
+    thiz->pool =
+        gst_msdkdec_create_buffer_pool (thiz, &thiz->output_info, min_buffers);
+    if (!thiz->pool)
+      goto failed_to_create_pool;
+  }
+
   if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)
       && gst_buffer_pool_has_option (pool,
           GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT)) {
@@ -880,20 +1123,10 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
     /* If downstream supports video meta and video alignment,
      * we can replace our own msdk bufferpool and use it
      */
-    GST_INFO_OBJECT (decoder, "create new MSDK bufferpool");
-
     /* Remove downstream's pool */
     gst_structure_free (pool_config);
     gst_object_unref (pool);
 
-    /* FIXME: this might break renegotiation.
-     * We should re-create msdk bufferpool, but it breaks decoding. */
-    if (!thiz->pool) {
-      thiz->pool =
-          gst_msdkdec_create_buffer_pool (thiz, pool_caps, min_buffers);
-      if (!thiz->pool)
-        goto failed_to_create_pool;
-    }
     pool = gst_object_ref (thiz->pool);
 
     /* Set the allocator of new msdk bufferpool */
@@ -903,31 +1136,24 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
       gst_query_set_nth_allocation_param (query, 0, allocator, NULL);
     gst_structure_free (config);
   } else {
-    /* If not, we just make a side-pool that will be decoded into and
-     * the copied from.
+    /* Unfortunately, dowstream doesn't have videometa or alignment support,
+     * we keep msdk pool as a side-pool that will be decoded into and
+     * then copied from.
      */
-    GST_INFO_OBJECT (decoder, "create new MSDK bufferpool as a side-pool");
-    thiz->pool =
-        gst_msdkdec_create_buffer_pool (thiz, pool_caps, thiz->async_depth);
-    if (!thiz->pool)
-      goto failed_to_create_pool;
+    GST_INFO_OBJECT (decoder, "Keep MSDK bufferpool as a side-pool");
 
     /* Update params to downstream's pool */
     gst_buffer_pool_config_set_params (pool_config, pool_caps, size,
         min_buffers, max_buffers);
     if (!gst_buffer_pool_set_config (pool, pool_config))
       goto error_set_config;
+    gst_video_info_from_caps (&thiz->non_msdk_pool_info, pool_caps);
 
-    /* Check if the pool's caps will meet msdk's alignment
-     * requirements by default and get aligned video info.
-     */
-    gst_video_info_from_caps (&info_from_caps, pool_caps);
-    info_aligned = info_from_caps;
-    gst_msdk_set_video_alignment (&info_from_caps, &alignment);
-    gst_video_info_align (&info_aligned, &alignment);
-
-    thiz->output_info = info_from_caps;
-    thiz->pool_info = info_aligned;
+    /* update width and height with actual negotiated values */
+    GST_VIDEO_INFO_WIDTH (&thiz->non_msdk_pool_info) =
+        GST_VIDEO_INFO_WIDTH (&thiz->output_info);
+    GST_VIDEO_INFO_HEIGHT (&thiz->non_msdk_pool_info) =
+        GST_VIDEO_INFO_HEIGHT (&thiz->output_info);
   }
 
   gst_query_set_nth_allocation_pool (query, 0, pool, size, min_buffers,
@@ -936,11 +1162,6 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
   if (pool)
     gst_object_unref (pool);
 
-  /* Initialize MSDK decoder before new bufferpool tries to alloc each buffer,
-   * which requires information of frame allocation.
-   */
-  if (!gst_msdkdec_init_decoder (thiz))
-    return FALSE;
 
   return TRUE;
 
@@ -1171,12 +1392,15 @@ static void
 gst_msdkdec_init (GstMsdkDec * thiz)
 {
   gst_video_info_init (&thiz->output_info);
-  gst_video_info_init (&thiz->pool_info);
+  gst_video_info_init (&thiz->non_msdk_pool_info);
   thiz->extra_params = g_ptr_array_new_with_free_func (g_free);
   thiz->tasks = g_array_new (FALSE, TRUE, sizeof (MsdkDecTask));
   thiz->hardware = PROP_HARDWARE_DEFAULT;
   thiz->async_depth = PROP_ASYNC_DEPTH_DEFAULT;
   thiz->output_order = PROP_OUTPUT_ORDER_DEFAULT;
   thiz->is_packetized = TRUE;
+  thiz->do_renego = TRUE;
+  thiz->do_realloc = TRUE;
+  thiz->force_reset_on_res_change = TRUE;
   thiz->adapter = gst_adapter_new ();
 }
diff --git a/sys/msdk/gstmsdkdec.h b/sys/msdk/gstmsdkdec.h
index f6f0825ed..6fe591502 100644
--- a/sys/msdk/gstmsdkdec.h
+++ b/sys/msdk/gstmsdkdec.h
@@ -62,9 +62,12 @@ struct _GstMsdkDec
 
   /* input description */
   GstVideoCodecState *input_state;
+  /* aligned msdk pool info */
   GstVideoInfo output_info;
   GstBufferPool *pool;
-  GstVideoInfo pool_info;
+  GstCaps *allocation_caps;
+  /* downstream pool info based on allocation query */
+  GstVideoInfo non_msdk_pool_info;
   mfxFrameAllocResponse alloc_resp;
   gboolean use_video_memory;
   gboolean use_dmabuf;
@@ -73,6 +76,16 @@ struct _GstMsdkDec
   /* for packetization */
   GstAdapter *adapter;
   gboolean is_packetized;
+  /* cap negotiation needed, allocation may or may not be required*/
+  gboolean do_renego;
+  /* re-allocation is mandatory if enabled */
+  gboolean do_realloc;
+  /* force reset on resolution change */
+  gboolean force_reset_on_res_change;
+  /* minimum number of buffers to be allocated, this should
+   * include downstream requirement, msdk suggestion and extra
+   * surface allocation for smooth display in render pipeline */
+  guint min_prealloc_buffers;
 
   /* MFX context */
   GstMsdkContext *context;
diff --git a/sys/msdk/gstmsdkvp9dec.c b/sys/msdk/gstmsdkvp9dec.c
index 8c87defd6..3c0d9054b 100644
--- a/sys/msdk/gstmsdkvp9dec.c
+++ b/sys/msdk/gstmsdkvp9dec.c
@@ -86,6 +86,7 @@ gst_msdkvp9dec_configure (GstMsdkDec * decoder)
   decoder->param.mfx.FrameInfo.Height =
       GST_ROUND_UP_16 (decoder->param.mfx.FrameInfo.CropH);
 
+  decoder->force_reset_on_res_change = FALSE;
   return TRUE;
 }