VA: Add the aux surface for gst buffer used by decoder.

The AV1 codec needs to support the film grain feature. When the film
grain feature is enabled, we need two surfaces as the output of the
decoded picture, one without film grain effect and the other one with
it. The first one acts as the reference and is needed for later pictures'
reconstruction, and the second one is the real display output.
So we need to attach another aux surface to the gst buffer/mem and make
that aux surface as the target of vaBeginPicture.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1636>

4 files changed, 144 insertions, 3 deletions

diff --git a/sys/va/gstvaallocator.c b/sys/va/gstvaallocator.c
index a2ca9f0c5..9ef3b891a 100644
--- a/sys/va/gstvaallocator.c
+++ b/sys/va/gstvaallocator.c
@@ -320,6 +320,19 @@ gst_va_drm_mod_quark (void)
   return drm_mod_quark;
 }
 
+static GQuark
+gst_va_buffer_aux_surface_quark (void)
+{
+  static gsize surface_quark = 0;
+
+  if (g_once_init_enter (&surface_quark)) {
+    GQuark quark = g_quark_from_string ("GstVaBufferAuxSurface");
+    g_once_init_leave (&surface_quark, quark);
+  }
+
+  return surface_quark;
+}
+
 /*========================= GstVaBufferSurface ===============================*/
 
 typedef struct _GstVaBufferSurface GstVaBufferSurface;
@@ -1605,3 +1618,104 @@ gst_va_buffer_get_surface (GstBuffer * buffer)
 
   return gst_va_memory_get_surface (mem);
 }
+
+gboolean
+gst_va_buffer_create_aux_surface (GstBuffer * buffer)
+{
+  GstMemory *mem;
+  VASurfaceID surface = VA_INVALID_ID;
+  GstVaDisplay *display = NULL;
+  GstVideoFormat format;
+  gint width, height;
+  GstVaBufferSurface *surface_buffer;
+
+  mem = gst_buffer_peek_memory (buffer, 0);
+  if (!mem)
+    return FALSE;
+
+  /* Already created it. */
+  surface_buffer = gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
+      gst_va_buffer_aux_surface_quark ());
+  if (surface_buffer)
+    return TRUE;
+
+  if (!mem->allocator)
+    return FALSE;
+
+  if (GST_IS_VA_DMABUF_ALLOCATOR (mem->allocator)) {
+    GstVaDmabufAllocator *self = GST_VA_DMABUF_ALLOCATOR (mem->allocator);
+    guint32 fourcc, rt_format;
+
+    format = GST_VIDEO_INFO_FORMAT (&self->info);
+    fourcc = gst_va_fourcc_from_video_format (format);
+    rt_format = gst_va_chroma_from_video_format (format);
+    if (fourcc == 0 || rt_format == 0) {
+      GST_ERROR_OBJECT (self, "Unsupported format: %s",
+          gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&self->info)));
+      return FALSE;
+    }
+
+    display = self->display;
+    width = GST_VIDEO_INFO_WIDTH (&self->info);
+    height = GST_VIDEO_INFO_HEIGHT (&self->info);
+    if (!_create_surfaces (self->display, rt_format, fourcc,
+            GST_VIDEO_INFO_WIDTH (&self->info),
+            GST_VIDEO_INFO_HEIGHT (&self->info), self->usage_hint, NULL,
+            &surface, 1))
+      return FALSE;
+  } else if (GST_IS_VA_ALLOCATOR (mem->allocator)) {
+    GstVaAllocator *self = GST_VA_ALLOCATOR (mem->allocator);
+
+    if (self->fourcc == 0 || self->rt_format == 0) {
+      GST_ERROR_OBJECT (self, "Unknown fourcc or chroma format");
+      return FALSE;
+    }
+
+    display = self->display;
+    width = GST_VIDEO_INFO_WIDTH (&self->info);
+    height = GST_VIDEO_INFO_HEIGHT (&self->info);
+    format = GST_VIDEO_INFO_FORMAT (&self->info);
+    if (!_create_surfaces (self->display, self->rt_format, self->fourcc,
+            GST_VIDEO_INFO_WIDTH (&self->info),
+            GST_VIDEO_INFO_HEIGHT (&self->info), self->usage_hint, NULL,
+            &surface, 1))
+      return FALSE;
+  } else {
+    g_assert_not_reached ();
+  }
+
+  if (!display || surface == VA_INVALID_ID)
+    return FALSE;
+
+  surface_buffer = gst_va_buffer_surface_new (surface, format, width, height);
+  surface_buffer->display = gst_object_ref (display);
+  g_atomic_int_add (&surface_buffer->ref_count, 1);
+
+  gst_mini_object_set_qdata (GST_MINI_OBJECT (mem),
+      gst_va_buffer_aux_surface_quark (), surface_buffer,
+      gst_va_buffer_surface_unref);
+
+  return TRUE;
+}
+
+VASurfaceID
+gst_va_buffer_get_aux_surface (GstBuffer * buffer)
+{
+  GstVaBufferSurface *surface_buffer;
+  GstMemory *mem;
+
+  mem = gst_buffer_peek_memory (buffer, 0);
+  if (!mem)
+    return VA_INVALID_ID;
+
+  surface_buffer = gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
+      gst_va_buffer_aux_surface_quark ());
+  if (!surface_buffer)
+    return VA_INVALID_ID;
+
+  /* No one increments it, and its lifetime is the same with the
+     gstmemory itself */
+  g_assert (g_atomic_int_get (&surface_buffer->ref_count) == 1);
+
+  return surface_buffer->surface;
+}
diff --git a/sys/va/gstvaallocator.h b/sys/va/gstvaallocator.h
index 23e06ee7e..e4123c251 100644
--- a/sys/va/gstvaallocator.h
+++ b/sys/va/gstvaallocator.h
@@ -77,4 +77,7 @@ gboolean              gst_va_allocator_get_format         (GstAllocator * alloca
 VASurfaceID           gst_va_memory_get_surface           (GstMemory * mem);
 VASurfaceID           gst_va_buffer_get_surface           (GstBuffer * buffer);
 
+gboolean              gst_va_buffer_create_aux_surface    (GstBuffer * buffer);
+VASurfaceID           gst_va_buffer_get_aux_surface       (GstBuffer * buffer);
+
 G_END_DECLS
diff --git a/sys/va/gstvadecoder.c b/sys/va/gstvadecoder.c
index a75503844..f47728258 100644
--- a/sys/va/gstvadecoder.c
+++ b/sys/va/gstvadecoder.c
@@ -575,18 +575,23 @@ gst_va_decoder_add_slice_buffer (GstVaDecoder * self, GstVaDecodePicture * pic,
 }
 
 gboolean
-gst_va_decoder_decode (GstVaDecoder * self, GstVaDecodePicture * pic)
+gst_va_decoder_decode_with_aux_surface (GstVaDecoder * self,
+    GstVaDecodePicture * pic, gboolean use_aux)
 {
   VADisplay dpy;
   VAStatus status;
-  VASurfaceID surface;
+  VASurfaceID surface = VA_INVALID_ID;
   gboolean ret = FALSE;
 
   g_return_val_if_fail (GST_IS_VA_DECODER (self), FALSE);
   g_return_val_if_fail (self->context != VA_INVALID_ID, FALSE);
   g_return_val_if_fail (pic, FALSE);
 
-  surface = gst_va_decode_picture_get_surface (pic);
+  if (use_aux) {
+    surface = gst_va_decode_picture_get_aux_surface (pic);
+  } else {
+    surface = gst_va_decode_picture_get_surface (pic);
+  }
   if (surface == VA_INVALID_ID) {
     GST_ERROR_OBJECT (self, "Decode picture without VASurfaceID");
     return FALSE;
@@ -648,6 +653,12 @@ fail_end_pic:
   }
 }
 
+gboolean
+gst_va_decoder_decode (GstVaDecoder * self, GstVaDecodePicture * pic)
+{
+  return gst_va_decoder_decode_with_aux_surface (self, pic, FALSE);
+}
+
 static gboolean
 _destroy_buffers (GstVaDecodePicture * pic)
 {
@@ -716,6 +727,15 @@ gst_va_decode_picture_get_surface (GstVaDecodePicture * pic)
   return gst_va_buffer_get_surface (pic->gstbuffer);
 }
 
+VASurfaceID
+gst_va_decode_picture_get_aux_surface (GstVaDecodePicture * pic)
+{
+  g_return_val_if_fail (pic, VA_INVALID_ID);
+  g_return_val_if_fail (pic->gstbuffer, VA_INVALID_ID);
+
+  return gst_va_buffer_get_aux_surface (pic->gstbuffer);
+}
+
 void
 gst_va_decode_picture_free (GstVaDecodePicture * pic)
 {
diff --git a/sys/va/gstvadecoder.h b/sys/va/gstvadecoder.h
index 1c683546f..c3a16a354 100644
--- a/sys/va/gstvadecoder.h
+++ b/sys/va/gstvadecoder.h
@@ -75,10 +75,14 @@ gboolean              gst_va_decoder_add_slice_buffer_with_n_params
                                                            gsize slice_size);
 gboolean              gst_va_decoder_decode               (GstVaDecoder * self,
                                                            GstVaDecodePicture * pic);
+gboolean              gst_va_decoder_decode_with_aux_surface (GstVaDecoder * self,
+                                                              GstVaDecodePicture * pic,
+                                                              gboolean use_aux);
 
 GstVaDecodePicture *  gst_va_decode_picture_new           (GstVaDecoder * self,
                                                            GstBuffer * buffer);
 VASurfaceID           gst_va_decode_picture_get_surface   (GstVaDecodePicture * pic);
+VASurfaceID           gst_va_decode_picture_get_aux_surface (GstVaDecodePicture * pic);
 void                  gst_va_decode_picture_free          (GstVaDecodePicture * pic);
 GstVaDecodePicture *  gst_va_decode_picture_dup           (GstVaDecodePicture * pic);