msdkdec: use bufferpool

1\ In decide_allocation, it makes its own msdk bufferpool.
  - If downstream supports video meta, it just replace it with the msdk
bufferpool.
  - If not, it uses the msdk bufferpool as a side pool, which will be
decoded into.
    and will copy it to downstream's bufferpool.

2\ Decide if using video memory or system memory.
  - This is not completed in this patch.
  - It might be decided in update_src_caps.
  - But tested for both system memory and video memory cases.

https://bugzilla.gnome.org/show_bug.cgi?id=790752

2 files changed, 349 insertions, 151 deletions

diff --git a/sys/msdk/gstmsdkdec.c b/sys/msdk/gstmsdkdec.c
index 7c7aeaa1d..111a6aef3 100644
--- a/sys/msdk/gstmsdkdec.c
+++ b/sys/msdk/gstmsdkdec.c
@@ -36,6 +36,9 @@
 #include <stdlib.h>
 
 #include "gstmsdkdec.h"
+#include "gstmsdkbufferpool.h"
+#include "gstmsdkvideomemory.h"
+#include "gstmsdksystemmemory.h"
 
 GST_DEBUG_CATEGORY_EXTERN (gst_msdkdec_debug);
 #define GST_CAT_DEFAULT gst_msdkdec_debug
@@ -65,7 +68,8 @@ G_DEFINE_TYPE (GstMsdkDec, gst_msdkdec, GST_TYPE_VIDEO_DECODER);
 
 typedef struct _MsdkSurface
 {
-  mfxFrameSurface1 surface;
+  mfxFrameSurface1 *surface;
+  GstBuffer *buf;
   GstVideoFrame data;
   GstVideoFrame copy;
 } MsdkSurface;
@@ -76,6 +80,7 @@ allocate_output_buffer (GstMsdkDec * thiz, GstBuffer ** buffer)
   GstFlowReturn flow;
   GstVideoCodecFrame *frame;
   GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz);
+  mfxFrameSurface1 *mfx_surface;
 
   frame = gst_video_decoder_get_oldest_frame (decoder);
   if (!frame) {
@@ -84,13 +89,30 @@ allocate_output_buffer (GstMsdkDec * thiz, GstBuffer ** buffer)
     else
       return GST_FLOW_ERROR;
   }
+
   if (!frame->output_buffer) {
+  retry:
     flow = gst_video_decoder_allocate_output_frame (decoder, frame);
     if (flow != GST_FLOW_OK) {
       gst_video_codec_frame_unref (frame);
       return flow;
     }
+
+    if (gst_msdk_is_msdk_buffer (frame->output_buffer)) {
+      mfx_surface = gst_msdk_get_surface_from_buffer (frame->output_buffer);
+      /* When using video memory, mfx surface is still locked even though
+       * it's finished by SyncOperation. There's no way to get notified when it gets unlocked.
+       * We should be keep these buffers and check if it's unlocked.
+       */
+      if (mfx_surface && mfx_surface->Data.Locked) {
+        thiz->locked_buffer =
+            g_list_append (thiz->locked_buffer, frame->output_buffer);
+        frame->output_buffer = NULL;
+        goto retry;
+      }
+    }
   }
+
   *buffer = gst_buffer_ref (frame->output_buffer);
   gst_buffer_replace (&frame->output_buffer, NULL);
   gst_video_codec_frame_unref (frame);
@@ -98,33 +120,20 @@ allocate_output_buffer (GstMsdkDec * thiz, GstBuffer ** buffer)
 }
 
 static void
-free_surface (gpointer surface)
+free_surface (GstMsdkDec * thiz, MsdkSurface * s)
 {
-  MsdkSurface *s = surface;
-
-  if (s->surface.Data.Locked)
-    /* MSDK is using the surface, defer unmapping/unreffing. */
-    return;
-
   if (s->copy.buffer) {
     gst_video_frame_unmap (&s->copy);
     gst_buffer_unref (s->copy.buffer);
-    s->copy.buffer = NULL;
   }
 
-  if (s->data.buffer) {
+  if (s->data.buffer)
     gst_video_frame_unmap (&s->data);
-    gst_buffer_unref (s->data.buffer);
-    s->data.buffer = NULL;
-  }
-}
 
-static void
-clear_surface (gpointer surface)
-{
-  MsdkSurface *s = surface;
-  s->surface.Data.Locked = 0;
-  free_surface (surface);
+  gst_buffer_unref (s->buf);
+
+  g_slice_free (MsdkSurface, s);
+  thiz->decoded_msdk_surfaces = g_list_remove (thiz->decoded_msdk_surfaces, s);
 }
 
 static MsdkSurface *
@@ -132,39 +141,48 @@ get_surface (GstMsdkDec * thiz, GstBuffer * buffer)
 {
   MsdkSurface *i;
 
-  for (i = (MsdkSurface *) thiz->surfaces->data;
-      i < (MsdkSurface *) thiz->surfaces->data + thiz->surfaces->len; i++) {
-    if (!i->surface.Data.Locked)
-      break;
-  }
-  if (i == (MsdkSurface *) thiz->surfaces->data + thiz->surfaces->len)
-    return NULL;
-
-  /* MSDK may have been using a surface for its own purposes and then
-     released it. Release any buffers still held and then
-     re-allocate */
-  free_surface (i);
+  i = g_slice_new0 (MsdkSurface);
 
-  if (!thiz->pool) {
-    if (!gst_video_frame_map (&i->data, &thiz->output_info, buffer,
-            GST_MAP_READWRITE))
-      goto failed_unref_buffer;
+  if (gst_msdk_is_msdk_buffer (buffer)) {
+    i->surface = gst_msdk_get_surface_from_buffer (buffer);
+    i->buf = buffer;
   } else {
+    /* Confirm to activate the side pool */
+    if (!gst_buffer_pool_is_active (thiz->pool) &&
+        !gst_buffer_pool_set_active (thiz->pool, TRUE)) {
+      g_slice_free (MsdkSurface, i);
+      return NULL;
+    }
+
     if (!gst_video_frame_map (&i->copy, &thiz->output_info, buffer,
             GST_MAP_WRITE))
       goto failed_unref_buffer;
+
+  retry:
     if (gst_buffer_pool_acquire_buffer (thiz->pool, &buffer,
             NULL) != GST_FLOW_OK)
       goto failed_unmap_copy;
+
+    if (gst_msdk_is_msdk_buffer (buffer)) {
+      i->surface = gst_msdk_get_surface_from_buffer (buffer);
+
+      /* When using video memory, mfx surface is still locked even though
+       * it's finished by SyncOperation. There's no way to get notified when it gets unlocked.
+       * We should keep these buffers and check if it's unlocked.
+       */
+      if (i->surface->Data.Locked) {
+        thiz->locked_buffer = g_list_append (thiz->locked_buffer, buffer);
+        goto retry;
+      }
+      i->buf = buffer;
+    }
+
     if (!gst_video_frame_map (&i->data, &thiz->pool_info, buffer,
             GST_MAP_READWRITE))
       goto failed_unref_buffer2;
   }
 
-  i->surface.Data.Y = GST_VIDEO_FRAME_PLANE_DATA (&i->data, 0);
-  i->surface.Data.UV = GST_VIDEO_FRAME_PLANE_DATA (&i->data, 1);
-  i->surface.Data.PitchLow = GST_VIDEO_FRAME_PLANE_STRIDE (&i->data, 0);
-
+  thiz->decoded_msdk_surfaces = g_list_append (thiz->decoded_msdk_surfaces, i);
   return i;
 
 failed_unref_buffer2:
@@ -174,9 +192,9 @@ failed_unmap_copy:
   gst_video_frame_unmap (&i->copy);
 failed_unref_buffer:
   gst_buffer_unref (buffer);
+  g_slice_free (MsdkSurface, i);
 
-  i->data.buffer = NULL;
-  i->copy.buffer = NULL;
+  GST_ERROR_OBJECT (thiz, "failed to handle buffer");
   return NULL;
 }
 
@@ -190,6 +208,9 @@ gst_msdkdec_close_decoder (GstMsdkDec * thiz)
 
   GST_DEBUG_OBJECT (thiz, "Closing decoder 0x%p", thiz->context);
 
+  if (thiz->use_video_memory)
+    gst_msdk_frame_free (thiz->context, &thiz->alloc_resp);
+
   status = MFXVideoDECODE_Close (gst_msdk_context_get_session (thiz->context));
   if (status != MFX_ERR_NONE && status != MFX_ERR_NOT_INITIALIZED) {
     GST_WARNING_OBJECT (thiz, "Decoder close failed (%s)",
@@ -197,7 +218,6 @@ gst_msdkdec_close_decoder (GstMsdkDec * thiz)
   }
 
   g_array_set_size (thiz->tasks, 0);
-  g_array_set_size (thiz->surfaces, 0);
   g_ptr_array_set_size (thiz->extra_params, 0);
 
   if (thiz->context)
@@ -213,7 +233,6 @@ gst_msdkdec_init_decoder (GstMsdkDec * thiz)
   mfxSession session;
   mfxStatus status;
   mfxFrameAllocRequest request;
-  guint i;
 
   if (!thiz->input_state) {
     GST_DEBUG_OBJECT (thiz, "Have no input state yet");
@@ -232,9 +251,17 @@ gst_msdkdec_init_decoder (GstMsdkDec * thiz)
 
   GST_OBJECT_LOCK (thiz);
 
-  thiz->param.AsyncDepth = thiz->async_depth;
-  thiz->param.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
+  if (thiz->use_video_memory) {
+    gst_msdk_set_frame_allocator (thiz->context);
+    thiz->param.IOPattern = MFX_IOPATTERN_OUT_VIDEO_MEMORY;
+  } else {
+    thiz->param.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
+  }
 
+  GST_INFO_OBJECT (thiz, "This MSDK decoder uses %s memory",
+      thiz->use_video_memory ? "video" : "system");
+
+  thiz->param.AsyncDepth = thiz->async_depth;
   thiz->param.mfx.FrameInfo.Width = GST_ROUND_UP_32 (info->width);
   thiz->param.mfx.FrameInfo.Height = GST_ROUND_UP_32 (info->height);
   thiz->param.mfx.FrameInfo.CropW = info->width;
@@ -285,15 +312,14 @@ gst_msdkdec_init_decoder (GstMsdkDec * thiz)
     goto failed;
   }
 
-  g_array_set_size (thiz->surfaces, 0);
-  g_array_set_size (thiz->surfaces, request.NumFrameSuggested);
-  for (i = 0; i < thiz->surfaces->len; i++) {
-    memcpy (&g_array_index (thiz->surfaces, MsdkSurface, i).surface.Info,
-        &thiz->param.mfx.FrameInfo, sizeof (mfxFrameInfo));
+  if (thiz->use_video_memory) {
+    request.Type |= MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
+    request.NumFrameSuggested += thiz->async_depth;
+    gst_msdk_frame_alloc (thiz->context, &request, &thiz->alloc_resp);
   }
 
-  GST_DEBUG_OBJECT (thiz, "Required %d surfaces (%d suggested), allocated %d",
-      request.NumFrameMin, request.NumFrameSuggested, thiz->surfaces->len);
+  GST_DEBUG_OBJECT (thiz, "Required %d surfaces (%d suggested)",
+      request.NumFrameMin, request.NumFrameSuggested);
 
   status = MFXVideoDECODE_Init (session, &thiz->param);
   if (status < MFX_ERR_NONE) {
@@ -345,12 +371,17 @@ gst_msdkdec_set_src_caps (GstMsdkDec * thiz)
 
   gst_msdk_set_video_alignment (&output_state->info, &align);
   gst_video_info_align (&output_state->info, &align);
-  memcpy (&thiz->output_info, &output_state->info, sizeof (GstVideoInfo));
+  thiz->output_info = output_state->info;
   if (output_state->caps)
     gst_caps_unref (output_state->caps);
   output_state->caps = gst_video_info_to_caps (&output_state->info);
   gst_video_codec_state_unref (output_state);
 
+  /* TODO: If downstream accepts msdk memory or dmabuf,
+   * this should be TRUE and using MFX_IOPATTERN_OUT_VIDEO_MEMORY
+   */
+  thiz->use_video_memory = FALSE;
+
   return TRUE;
 }
 
@@ -381,6 +412,15 @@ gst_msdkdec_set_latency (GstMsdkDec * thiz)
   gst_video_decoder_set_latency (GST_VIDEO_DECODER (thiz), latency, latency);
 }
 
+static gint
+_find_msdk_surface (gconstpointer msdk_surface, gconstpointer comp_surface)
+{
+  MsdkSurface *cached_surface = (MsdkSurface *) msdk_surface;
+  mfxFrameSurface1 *_surface = (mfxFrameSurface1 *) comp_surface;
+
+  return cached_surface ? cached_surface->surface != _surface : -1;
+}
+
 static GstFlowReturn
 gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task)
 {
@@ -389,34 +429,45 @@ gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task)
   GstVideoCodecFrame *frame;
   MsdkSurface *surface;
   mfxStatus status;
+  GList *l;
 
   if (G_LIKELY (task->sync_point)) {
     status =
         MFXVideoCORE_SyncOperation (gst_msdk_context_get_session
-        (thiz->context), task->sync_point, 10000);
-    if (status != MFX_ERR_NONE)
+        (thiz->context), task->sync_point, 300000);
+    if (status != MFX_ERR_NONE) {
+      GST_ERROR_OBJECT (thiz, "failed to do sync operation");
       return GST_FLOW_ERROR;
-    frame = gst_video_decoder_get_oldest_frame (decoder);
+    }
 
+    frame = gst_video_decoder_get_oldest_frame (decoder);
     task->sync_point = NULL;
-    task->surface->Data.Locked--;
-    surface = (MsdkSurface *) task->surface;
+
+    l = g_list_find_custom (thiz->decoded_msdk_surfaces, task->surface,
+        _find_msdk_surface);
+    if (l) {
+      surface = l->data;
+    } else {
+      GST_ERROR_OBJECT (thiz, "Couldn't find the cached MSDK surface");
+      return GST_FLOW_ERROR;
+    }
 
     if (G_LIKELY (frame)) {
       if (G_LIKELY (surface->copy.buffer == NULL)) {
-        frame->output_buffer = gst_buffer_ref (surface->data.buffer);
+        frame->output_buffer = gst_buffer_ref (surface->buf);
       } else {
         gst_video_frame_copy (&surface->copy, &surface->data);
         frame->output_buffer = gst_buffer_ref (surface->copy.buffer);
       }
     }
 
-    free_surface (surface);
+    free_surface (thiz, surface);
 
     if (!frame)
       return GST_FLOW_FLUSHING;
-    flow = gst_video_decoder_finish_frame (decoder, frame);
     gst_video_codec_frame_unref (frame);
+
+    flow = gst_video_decoder_finish_frame (decoder, frame);
     return flow;
   }
   return GST_FLOW_OK;
@@ -459,19 +510,57 @@ gst_msdkdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
     gst_video_codec_state_unref (thiz->input_state);
   thiz->input_state = gst_video_codec_state_ref (state);
 
-  if (!gst_msdkdec_init_decoder (thiz))
+  if (!gst_msdkdec_set_src_caps (thiz))
     return FALSE;
 
-  if (!gst_msdkdec_set_src_caps (thiz)) {
-    gst_msdkdec_close_decoder (thiz);
+  if (!gst_msdkdec_init_decoder (thiz))
     return FALSE;
-  }
 
   gst_msdkdec_set_latency (thiz);
-
   return TRUE;
 }
 
+static void
+release_msdk_surfaces (GstMsdkDec * thiz)
+{
+  GList *l;
+  MsdkSurface *surface;
+
+  for (l = thiz->decoded_msdk_surfaces; l; l = l->next) {
+    surface = l->data;
+    free_surface (thiz, surface);
+  }
+}
+
+static void
+release_locked_buffer (GstMsdkDec * thiz)
+{
+  GList *l;
+  GstBuffer *buf;
+
+  for (l = thiz->locked_buffer; l; l = l->next) {
+    buf = l->data;
+    gst_buffer_unref (buf);
+  }
+}
+
+static void
+check_locked_buffer (GstMsdkDec * thiz)
+{
+  GList *l;
+  GstBuffer *buf;
+  mfxFrameSurface1 *mfx_surface;
+
+  for (l = thiz->locked_buffer; l; l = l->next) {
+    buf = l->data;
+    mfx_surface = gst_msdk_get_surface_from_buffer (buf);
+    if (!mfx_surface->Data.Locked) {
+      gst_buffer_unref (buf);
+      thiz->locked_buffer = g_list_delete_link (thiz->locked_buffer, l);
+    }
+  }
+}
+
 static GstFlowReturn
 gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
 {
@@ -479,8 +568,8 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
   GstFlowReturn flow;
   GstBuffer *buffer;
   MsdkDecTask *task = NULL;
-  MsdkSurface *surface = NULL;
   mfxBitstream bitstream;
+  MsdkSurface *surface = NULL;
   mfxSession session;
   mfxStatus status;
   GstMapInfo map_info;
@@ -488,6 +577,7 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
 
   if (!gst_buffer_map (frame->input_buffer, &map_info, GST_MAP_READ))
     return GST_FLOW_ERROR;
+
   memset (&bitstream, 0, sizeof (bitstream));
   bitstream.Data = map_info.data;
   bitstream.DataLength = map_info.size;
@@ -495,6 +585,8 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
 
   session = gst_msdk_context_get_session (thiz->context);
   for (;;) {
+    check_locked_buffer (thiz);
+
     task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
     flow = gst_msdkdec_finish_task (thiz, task);
     if (flow != GST_FLOW_OK)
@@ -526,32 +618,35 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
     }
 
     status =
-        MFXVideoDECODE_DecodeFrameAsync (session, &bitstream, &surface->surface,
+        MFXVideoDECODE_DecodeFrameAsync (session, &bitstream, surface->surface,
         &task->surface, &task->sync_point);
     if (G_LIKELY (status == MFX_ERR_NONE)) {
-      /* Locked may not be incremented immediately by the SDK, but
-         this surface should not be given as a work surface again
-         until after SyncOperation has been called. We may loop right
-         back up to get_surface, if more bitstream is available to
-         handle.  So increment Locked ourselves and then decrement it
-         after SyncOperation. */
-      task->surface->Data.Locked++;
       thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
-      surface = NULL;
+
+      if (surface->surface->Data.Locked > 0 || !thiz->use_video_memory)
+        surface = NULL;
+
       if (bitstream.DataLength == 0) {
         flow = GST_FLOW_OK;
         break;
       }
     } else if (status == MFX_ERR_MORE_DATA) {
+      if (surface->surface->Data.Locked > 0)
+        surface = NULL;
       flow = GST_FLOW_OK;
       break;
     } else if (status == MFX_ERR_MORE_SURFACE) {
       surface = NULL;
       continue;
-    } else if (status == MFX_WRN_DEVICE_BUSY)
+    } else if (status == MFX_WRN_DEVICE_BUSY) {
       /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */
       g_usleep (1000);
-    else if (status < MFX_ERR_NONE) {
+
+      /* If the current surface is still busy, we should do sync oepration
+       * then tries to decode again
+       */
+      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
+    } else if (status < MFX_ERR_NONE) {
       GST_ERROR_OBJECT (thiz, "DecodeFrameAsync failed (%s)",
           msdk_status_to_string (status));
       flow = GST_FLOW_ERROR;
@@ -561,12 +656,82 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame)
 
 exit:
   if (surface)
-    free_surface (surface);
+    free_surface (thiz, surface);
 
   gst_buffer_unmap (frame->input_buffer, &map_info);
   return flow;
 }
 
+static GstBufferPool *
+gst_msdkdec_create_buffer_pool (GstMsdkDec * thiz, GstCaps * caps,
+    guint num_buffers)
+{
+  GstBufferPool *pool = NULL;
+  GstStructure *config;
+  GstAllocator *allocator = NULL;
+  GstVideoInfo info;
+  GstVideoAlignment align;
+  GstAllocationParams params = { 0, 31, 0, 0, };
+  mfxFrameAllocResponse *alloc_resp = NULL;
+
+  alloc_resp = &thiz->alloc_resp;
+
+  pool = gst_msdk_buffer_pool_new (thiz->context, alloc_resp);
+  if (!pool)
+    goto error_no_pool;
+
+  if (!gst_video_info_from_caps (&info, caps)) {
+    GST_INFO_OBJECT (thiz, "failed to get video info");
+    return FALSE;
+  }
+
+  gst_msdk_set_video_alignment (&info, &align);
+  gst_video_info_align (&info, &align);
+
+  if (thiz->use_video_memory)
+    allocator = gst_msdk_video_allocator_new (thiz->context, &info, alloc_resp);
+  else
+    allocator = gst_msdk_system_allocator_new (&info);
+
+  if (!allocator)
+    goto error_no_allocator;
+
+  config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));
+  gst_buffer_pool_config_set_params (config, caps, info.size, num_buffers, 0);
+  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
+  gst_buffer_pool_config_add_option (config,
+      GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
+
+  if (thiz->use_video_memory)
+    gst_buffer_pool_config_add_option (config,
+        GST_BUFFER_POOL_OPTION_MSDK_USE_VIDEO_MEMORY);
+
+  gst_buffer_pool_config_set_video_alignment (config, &align);
+  gst_buffer_pool_config_set_allocator (config, allocator, &params);
+  gst_object_unref (allocator);
+
+  if (!gst_buffer_pool_set_config (pool, config))
+    goto error_pool_config;
+
+  return pool;
+
+error_no_pool:
+  {
+    GST_INFO_OBJECT (thiz, "failed to create bufferpool");
+    return NULL;
+  }
+error_no_allocator:
+  {
+    GST_INFO_OBJECT (thiz, "failed to create allocator");
+    return NULL;
+  }
+error_pool_config:
+  {
+    GST_INFO_OBJECT (thiz, "failed to set config");
+    return NULL;
+  }
+}
+
 static gboolean
 gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
 {
@@ -576,7 +741,6 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
   GstBufferPool *pool = NULL;
   GstStructure *pool_config = NULL;
   GstCaps *pool_caps;
-  gboolean need_aligned;
   guint size, min_buffers, max_buffers;
 
   if (!GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation (decoder,
@@ -589,62 +753,71 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
   gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL);
   pool_config = gst_buffer_pool_get_config (pool);
 
-  /* Increase the min and max buffers by async_depth, we will
-     always have that number of decode operations in-flight */
+  /* Get the caps of pool and increase the min and max buffers by async_depth,
+   * we will always have that number of decode operations in-flight */
   gst_buffer_pool_config_get_params (pool_config, &pool_caps, &size,
       &min_buffers, &max_buffers);
   min_buffers += thiz->async_depth;
   if (max_buffers)
     max_buffers += thiz->async_depth;
-  gst_buffer_pool_config_set_params (pool_config, pool_caps, size, min_buffers,
-      max_buffers);
 
-  /* Check if the pool's caps will meet msdk's alignment
-     requirements by default. */
-  gst_video_info_from_caps (&info_from_caps, pool_caps);
-  memcpy (&info_aligned, &info_from_caps, sizeof (info_aligned));
-  gst_msdk_set_video_alignment (&info_from_caps, &alignment);
-  gst_video_info_align (&info_aligned, &alignment);
-  need_aligned = !gst_video_info_is_equal (&info_from_caps, &info_aligned);
-
-  if (need_aligned) {
-    /* The pool's caps do not meet msdk's alignment requirements. Make
-       a pool config that does meet the requirements. We will use this
-       config for the allocation pool if possible, or as the config
-       for a side-pool if the downstream can't handle it. */
-
-    size = MAX (size, GST_VIDEO_INFO_SIZE (&info_aligned));
+  if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)
+      && gst_buffer_pool_has_option (pool,
+          GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT)) {
+    GstStructure *config;
+    GstAllocator *allocator;
+
+    /* If downstream supports video meta and video alignment,
+     * we can replace our own msdk bufferpool and use it
+     */
+    GST_INFO_OBJECT (decoder, "create new MSDK bufferpool");
+
+    /* Remove downstream's pool */
+    gst_structure_free (pool_config);
+    gst_object_unref (pool);
+
+    /* FIXME: this might break renegotiation.
+     * We should re-create msdk bufferpool, but it breaks decoding. */
+    if (!thiz->pool) {
+      thiz->pool =
+          gst_msdkdec_create_buffer_pool (thiz, pool_caps, min_buffers);
+      if (!thiz->pool)
+        goto failed_to_create_pool;
+    }
+    pool = gst_object_ref (thiz->pool);
+
+    /* Set the allocator of new msdk bufferpool */
+    config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool));
+
+    if (gst_buffer_pool_config_get_allocator (config, &allocator, NULL))
+      gst_query_set_nth_allocation_param (query, 0, allocator, NULL);
+    gst_structure_free (config);
+  } else {
+    /* If not, we just make a side-pool that will be decoded into and
+     * the copied from.
+     */
+    GST_INFO_OBJECT (decoder, "create new MSDK bufferpool as a side-pool");
+    thiz->pool =
+        gst_msdkdec_create_buffer_pool (thiz, pool_caps, thiz->async_depth);
+    if (!thiz->pool)
+      goto failed_to_create_pool;
+
+    /* Update params to downstream's pool */
     gst_buffer_pool_config_set_params (pool_config, pool_caps, size,
         min_buffers, max_buffers);
-    gst_buffer_pool_config_add_option (pool_config,
-        GST_BUFFER_POOL_OPTION_VIDEO_META);
-    gst_buffer_pool_config_add_option (pool_config,
-        GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
-    gst_buffer_pool_config_set_video_alignment (pool_config, &alignment);
-
-    if (thiz->pool)
-      gst_object_unref (thiz->pool);
-    thiz->pool = NULL;
+    if (!gst_buffer_pool_set_config (pool, pool_config))
+      goto error_set_config;
 
-    if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)
-        && gst_buffer_pool_has_option (pool,
-            GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT)) {
-      /* The aligned pool config can be used directly. */
-      if (!gst_buffer_pool_set_config (pool, pool_config))
-        goto error_set_config;
-    } else {
-      /* The aligned pool config cannot be used directly so we will
-         make a side-pool that will be decoded into and the copied
-         from. */
-      thiz->pool = gst_video_buffer_pool_new ();
-      gst_buffer_pool_config_set_params (pool_config, pool_caps, size,
-          thiz->async_depth, max_buffers);
-      memcpy (&thiz->output_info, &info_from_caps, sizeof (GstVideoInfo));
-      memcpy (&thiz->pool_info, &info_aligned, sizeof (GstVideoInfo));
-      if (!gst_buffer_pool_set_config (thiz->pool, pool_config) ||
-          !gst_buffer_pool_set_active (thiz->pool, TRUE))
-        goto error_set_config;
-    }
+    /* Check if the pool's caps will meet msdk's alignment
+     * requirements by default and get aligned video info.
+     */
+    gst_video_info_from_caps (&info_from_caps, pool_caps);
+    info_aligned = info_from_caps;
+    gst_msdk_set_video_alignment (&info_from_caps, &alignment);
+    gst_video_info_align (&info_aligned, &alignment);
+
+    thiz->output_info = info_from_caps;
+    thiz->pool_info = info_aligned;
   }
 
   gst_query_set_nth_allocation_pool (query, 0, pool, size, min_buffers,
@@ -655,19 +828,17 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
 
   return TRUE;
 
-error_set_config:
+failed_to_create_pool:
   GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
   if (pool)
     gst_object_unref (pool);
   return FALSE;
-}
-
-static gboolean
-gst_msdkdec_flush (GstVideoDecoder * decoder)
-{
-  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
 
-  return gst_msdkdec_init_decoder (thiz);
+error_set_config:
+  GST_ERROR_OBJECT (decoder, "failed to set buffer pool config");
+  if (pool)
+    gst_object_unref (pool);
+  return FALSE;
 }
 
 static GstFlowReturn
@@ -687,11 +858,14 @@ gst_msdkdec_drain (GstVideoDecoder * decoder)
   session = gst_msdk_context_get_session (thiz->context);
 
   for (;;) {
+    check_locked_buffer (thiz);
+
     task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
-    if (gst_msdkdec_finish_task (thiz, task) != GST_FLOW_OK) {
-      GST_WARNING_OBJECT (decoder,
-          "failed to finish the task %p, but keep draining for the remaining frames",
-          task);
+    if ((flow = gst_msdkdec_finish_task (thiz, task)) != GST_FLOW_OK) {
+      if (flow != GST_FLOW_FLUSHING)
+        GST_WARNING_OBJECT (decoder,
+            "failed to finish the task %p, but keep draining for the remaining frames",
+            task);
     }
 
     if (!surface) {
@@ -704,24 +878,34 @@ gst_msdkdec_drain (GstVideoDecoder * decoder)
     }
 
     status =
-        MFXVideoDECODE_DecodeFrameAsync (session, NULL, &surface->surface,
+        MFXVideoDECODE_DecodeFrameAsync (session, NULL, surface->surface,
         &task->surface, &task->sync_point);
     if (G_LIKELY (status == MFX_ERR_NONE)) {
-      task->surface->Data.Locked++;
       thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
+
+      if (surface->surface->Data.Locked == 0)
+        free_surface (thiz, surface);
       surface = NULL;
-    } else if (status == MFX_WRN_VIDEO_PARAM_CHANGED)
+    } else if (status == MFX_WRN_VIDEO_PARAM_CHANGED) {
       continue;
-    else if (status == MFX_WRN_DEVICE_BUSY) {
+    } else if (status == MFX_WRN_DEVICE_BUSY) {
       /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */
       g_usleep (1000);
-    } else if (status == MFX_ERR_MORE_DATA)
+
+      /* If the current surface is still busy, we should do sync oepration
+       * then tries to decode again
+       */
+      thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
+    } else if (status == MFX_ERR_MORE_DATA) {
       break;
-    else if (status < MFX_ERR_NONE)
+    } else if (status == MFX_ERR_MORE_SURFACE) {
+      surface = NULL;
+      continue;
+    } else if (status < MFX_ERR_NONE)
       return GST_FLOW_ERROR;
   }
   if (surface)
-    free_surface (surface);
+    free_surface (thiz, surface);
 
   for (i = 0; i < thiz->tasks->len; i++) {
     task = &g_array_index (thiz->tasks, MsdkDecTask, thiz->next_task);
@@ -730,9 +914,22 @@ gst_msdkdec_drain (GstVideoDecoder * decoder)
       return flow;
     thiz->next_task = (thiz->next_task + 1) % thiz->tasks->len;
   }
+
+  check_locked_buffer (thiz);
+  release_locked_buffer (thiz);
+  release_msdk_surfaces (thiz);
+
   return GST_FLOW_OK;
 }
 
+static gboolean
+gst_msdkdec_flush (GstVideoDecoder * decoder)
+{
+  GstMsdkDec *thiz = GST_MSDKDEC (decoder);
+
+  return gst_msdkdec_drain (GST_VIDEO_DECODER_CAST (thiz));
+}
+
 static GstFlowReturn
 gst_msdkdec_finish (GstVideoDecoder * decoder)
 {
@@ -801,7 +998,6 @@ gst_msdkdec_finalize (GObject * object)
 {
   GstMsdkDec *thiz = GST_MSDKDEC (object);
 
-  g_array_unref (thiz->surfaces);
   g_array_unref (thiz->tasks);
   g_ptr_array_unref (thiz->extra_params);
 }
@@ -850,8 +1046,6 @@ gst_msdkdec_init (GstMsdkDec * thiz)
   gst_video_info_init (&thiz->output_info);
   gst_video_info_init (&thiz->pool_info);
   thiz->extra_params = g_ptr_array_new_with_free_func (g_free);
-  thiz->surfaces = g_array_new (FALSE, TRUE, sizeof (MsdkSurface));
-  g_array_set_clear_func (thiz->surfaces, clear_surface);
   thiz->tasks = g_array_new (FALSE, TRUE, sizeof (MsdkDecTask));
   thiz->hardware = PROP_HARDWARE_DEFAULT;
   thiz->async_depth = PROP_ASYNC_DEPTH_DEFAULT;
diff --git a/sys/msdk/gstmsdkdec.h b/sys/msdk/gstmsdkdec.h
index 517dc1563..ec386daaf 100644
--- a/sys/msdk/gstmsdkdec.h
+++ b/sys/msdk/gstmsdkdec.h
@@ -65,15 +65,19 @@ struct _GstMsdkDec
   GstVideoInfo output_info;
   GstBufferPool *pool;
   GstVideoInfo pool_info;
+  mfxFrameAllocResponse alloc_resp;
+  gboolean use_video_memory;
 
   /* MFX context */
   GstMsdkContext *context;
   mfxVideoParam param;
   GPtrArray *extra_params;
-  GArray *surfaces;
   GArray *tasks;
   guint next_task;
 
+  GList *decoded_msdk_surfaces;
+  GList *locked_buffer;
+
   /* element properties */
   gboolean hardware;
   guint async_depth;