codecs: AV1decoder: Add the AV1 decoder base class.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1636>

5 files changed, 1146 insertions, 0 deletions

diff --git a/gst-libs/gst/codecs/gstav1decoder.c b/gst-libs/gst/codecs/gstav1decoder.c
new file mode 100644
index 000000000..bf7cb64be
--- /dev/null
+++ b/gst-libs/gst/codecs/gstav1decoder.c
@@ -0,0 +1,614 @@
+/* GStreamer
+ * Copyright (C) 2020 He Junyan <junyan.he@intel.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * SECTION:gstav1decoder
+ * @title: Gstav1Decoder
+ * @short_description: Base class to implement stateless AV1 decoders
+ * @sources:
+ * - gstav1picture.h
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "gstav1decoder.h"
+
+GST_DEBUG_CATEGORY (gst_av1_decoder_debug);
+#define GST_CAT_DEFAULT gst_av1_decoder_debug
+
+struct _GstAV1DecoderPrivate
+{
+  gint max_width;
+  gint max_height;
+  GstAV1Profile profile;
+  GstAV1Parser *parser;
+  GstAV1Dpb *dpb;
+  GstAV1Picture *current_picture;
+  GstVideoCodecFrame *current_frame;
+};
+
+#define parent_class gst_av1_decoder_parent_class
+G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstAV1Decoder, gst_av1_decoder,
+    GST_TYPE_VIDEO_DECODER,
+    G_ADD_PRIVATE (GstAV1Decoder);
+    GST_DEBUG_CATEGORY_INIT (gst_av1_decoder_debug, "av1decoder", 0,
+        "AV1 Video Decoder"));
+
+static gboolean gst_av1_decoder_start (GstVideoDecoder * decoder);
+static gboolean gst_av1_decoder_stop (GstVideoDecoder * decoder);
+static gboolean gst_av1_decoder_set_format (GstVideoDecoder * decoder,
+    GstVideoCodecState * state);
+static GstFlowReturn gst_av1_decoder_finish (GstVideoDecoder * decoder);
+static gboolean gst_av1_decoder_flush (GstVideoDecoder * decoder);
+static GstFlowReturn gst_av1_decoder_drain (GstVideoDecoder * decoder);
+static GstFlowReturn gst_av1_decoder_handle_frame (GstVideoDecoder * decoder,
+    GstVideoCodecFrame * frame);
+
+static GstAV1Picture *gst_av1_decoder_duplicate_picture_default (GstAV1Decoder *
+    decoder, GstAV1Picture * picture);
+
+static void
+gst_av1_decoder_class_init (GstAV1DecoderClass * klass)
+{
+  GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
+
+  decoder_class->start = GST_DEBUG_FUNCPTR (gst_av1_decoder_start);
+  decoder_class->stop = GST_DEBUG_FUNCPTR (gst_av1_decoder_stop);
+  decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_av1_decoder_set_format);
+  decoder_class->finish = GST_DEBUG_FUNCPTR (gst_av1_decoder_finish);
+  decoder_class->flush = GST_DEBUG_FUNCPTR (gst_av1_decoder_flush);
+  decoder_class->drain = GST_DEBUG_FUNCPTR (gst_av1_decoder_drain);
+  decoder_class->handle_frame =
+      GST_DEBUG_FUNCPTR (gst_av1_decoder_handle_frame);
+
+  klass->duplicate_picture =
+      GST_DEBUG_FUNCPTR (gst_av1_decoder_duplicate_picture_default);
+}
+
+static void
+gst_av1_decoder_init (GstAV1Decoder * self)
+{
+  gst_video_decoder_set_packetized (GST_VIDEO_DECODER (self), TRUE);
+
+  self->priv = gst_av1_decoder_get_instance_private (self);
+}
+
+static void
+gst_av1_decoder_reset (GstAV1Decoder * self)
+{
+  GstAV1DecoderPrivate *priv = self->priv;
+
+  priv->max_width = 0;
+  priv->max_height = 0;
+  gst_av1_picture_clear (&priv->current_picture);
+  priv->current_frame = NULL;
+  priv->profile = GST_AV1_PROFILE_UNDEFINED;
+
+  if (priv->dpb)
+    gst_av1_dpb_clear (priv->dpb);
+  if (priv->parser)
+    gst_av1_parser_reset (priv->parser, FALSE);
+}
+
+static gboolean
+gst_av1_decoder_start (GstVideoDecoder * decoder)
+{
+  GstAV1Decoder *self = GST_AV1_DECODER (decoder);
+  GstAV1DecoderPrivate *priv = self->priv;
+
+  priv->parser = gst_av1_parser_new ();
+  priv->dpb = gst_av1_dpb_new ();
+
+  gst_av1_decoder_reset (self);
+
+  return TRUE;
+}
+
+static gboolean
+gst_av1_decoder_stop (GstVideoDecoder * decoder)
+{
+  GstAV1Decoder *self = GST_AV1_DECODER (decoder);
+  GstAV1DecoderPrivate *priv = self->priv;
+
+  gst_av1_decoder_reset (self);
+
+  g_clear_pointer (&self->input_state, gst_video_codec_state_unref);
+  g_clear_pointer (&priv->parser, gst_av1_parser_free);
+  g_clear_pointer (&priv->dpb, gst_av1_dpb_free);
+
+  return TRUE;
+}
+
+static gboolean
+gst_av1_decoder_set_format (GstVideoDecoder * decoder,
+    GstVideoCodecState * state)
+{
+  GstAV1Decoder *self = GST_AV1_DECODER (decoder);
+  GstAV1DecoderPrivate *priv = self->priv;
+
+  GST_DEBUG_OBJECT (decoder, "Set format");
+
+  if (self->input_state)
+    gst_video_codec_state_unref (self->input_state);
+
+  self->input_state = gst_video_codec_state_ref (state);
+
+  priv->max_width = GST_VIDEO_INFO_WIDTH (&state->info);
+  priv->max_height = GST_VIDEO_INFO_HEIGHT (&state->info);
+
+  return TRUE;
+}
+
+static GstFlowReturn
+gst_av1_decoder_finish (GstVideoDecoder * decoder)
+{
+  GST_DEBUG_OBJECT (decoder, "finish");
+
+  gst_av1_decoder_reset (GST_AV1_DECODER (decoder));
+
+  return GST_FLOW_OK;
+}
+
+static gboolean
+gst_av1_decoder_flush (GstVideoDecoder * decoder)
+{
+  GST_DEBUG_OBJECT (decoder, "flush");
+
+  gst_av1_decoder_reset (GST_AV1_DECODER (decoder));
+
+  return TRUE;
+}
+
+static GstFlowReturn
+gst_av1_decoder_drain (GstVideoDecoder * decoder)
+{
+  GST_DEBUG_OBJECT (decoder, "drain");
+
+  gst_av1_decoder_reset (GST_AV1_DECODER (decoder));
+
+  return GST_FLOW_OK;
+}
+
+static GstAV1Picture *
+gst_av1_decoder_duplicate_picture_default (GstAV1Decoder * decoder,
+    GstAV1Picture * picture)
+{
+  GstAV1Picture *new_picture;
+
+  new_picture = gst_av1_picture_new ();
+
+  return new_picture;
+}
+
+static const gchar *
+get_obu_name (GstAV1OBUType type)
+{
+  switch (type) {
+    case GST_AV1_OBU_SEQUENCE_HEADER:
+      return "sequence header";
+    case GST_AV1_OBU_TEMPORAL_DELIMITER:
+      return "temporal delimiter";
+    case GST_AV1_OBU_FRAME_HEADER:
+      return "frame header";
+    case GST_AV1_OBU_TILE_GROUP:
+      return "tile group";
+    case GST_AV1_OBU_METADATA:
+      return "metadata";
+    case GST_AV1_OBU_FRAME:
+      return "frame";
+    case GST_AV1_OBU_REDUNDANT_FRAME_HEADER:
+      return "redundant frame header";
+    case GST_AV1_OBU_TILE_LIST:
+      return "tile list";
+    case GST_AV1_OBU_PADDING:
+      return "padding";
+    default:
+      return "unknown";
+  }
+
+  return NULL;
+}
+
+static const gchar *
+gst_av1_decoder_profile_to_string (GstAV1Profile profile)
+{
+  switch (profile) {
+    case GST_AV1_PROFILE_0:
+      return "0";
+    case GST_AV1_PROFILE_1:
+      return "1";
+    case GST_AV1_PROFILE_2:
+      return "2";
+    default:
+      break;
+  }
+
+  return NULL;
+}
+
+static gboolean
+gst_av1_decoder_process_sequence (GstAV1Decoder * self, GstAV1OBU * obu)
+{
+  GstAV1ParserResult res;
+  GstAV1DecoderPrivate *priv = self->priv;
+  GstAV1SequenceHeaderOBU seq_header;
+  GstAV1SequenceHeaderOBU old_seq_header = { 0, };
+  GstAV1DecoderClass *klass = GST_AV1_DECODER_GET_CLASS (self);
+
+  if (priv->parser->seq_header)
+    old_seq_header = *priv->parser->seq_header;
+
+  res = gst_av1_parser_parse_sequence_header_obu (priv->parser,
+      obu, &seq_header);
+  if (res != GST_AV1_PARSER_OK) {
+    GST_WARNING_OBJECT (self, "Parsing sequence failed.");
+    return FALSE;
+  }
+
+  if (!memcmp (&old_seq_header, &seq_header, sizeof (GstAV1SequenceHeaderOBU))) {
+    GST_DEBUG_OBJECT (self, "Get same sequence header.");
+    return TRUE;
+  }
+
+  g_assert (klass->new_sequence);
+
+  GST_DEBUG_OBJECT (self,
+      "Sequence updated, profile %s -> %s, max resolution: %dx%d -> %dx%d",
+      gst_av1_decoder_profile_to_string (priv->profile),
+      gst_av1_decoder_profile_to_string (seq_header.seq_profile),
+      priv->max_width, priv->max_height, seq_header.max_frame_width_minus_1 + 1,
+      seq_header.max_frame_width_minus_1 + 1);
+
+  if (!klass->new_sequence (self, &seq_header)) {
+    GST_ERROR_OBJECT (self, "subclass does not want accept new sequence");
+    return FALSE;
+  }
+
+  priv->profile = seq_header.seq_profile;
+  priv->max_width = seq_header.max_frame_width_minus_1 + 1;
+  priv->max_height = seq_header.max_frame_height_minus_1 + 1;
+  gst_av1_dpb_clear (priv->dpb);
+
+  return TRUE;
+}
+
+static gboolean
+gst_av1_decoder_decode_tile_group (GstAV1Decoder * self,
+    GstAV1TileGroupOBU * tile_group, GstAV1OBU * obu)
+{
+  GstAV1DecoderPrivate *priv = self->priv;
+  GstAV1DecoderClass *klass = GST_AV1_DECODER_GET_CLASS (self);
+  GstAV1Picture *picture = priv->current_picture;
+  GstAV1Tile tile;
+
+  if (!picture) {
+    GST_ERROR_OBJECT (self, "No picture has created for current frame");
+    return FALSE;
+  }
+
+  if (picture->frame_hdr.show_existing_frame) {
+    GST_ERROR_OBJECT (self, "Current picture is showing the existing frame.");
+    return FALSE;
+  }
+
+  tile.obu = *obu;
+  tile.tile_group = *tile_group;
+
+  g_assert (klass->decode_tile);
+  if (!klass->decode_tile (self, picture, &tile)) {
+    GST_ERROR_OBJECT (self, "Decode tile error");
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
+gst_av1_decoder_decode_frame_header (GstAV1Decoder * self,
+    GstAV1FrameHeaderOBU * frame_header)
+{
+  GstAV1DecoderPrivate *priv = self->priv;
+  GstAV1DecoderClass *klass = GST_AV1_DECODER_GET_CLASS (self);
+  GstAV1Picture *picture = NULL;
+
+  g_assert (priv->current_frame);
+
+  if (priv->current_picture != NULL) {
+    GST_ERROR_OBJECT (self, "Already have picture for current frame");
+    return FALSE;
+  }
+
+  if (frame_header->show_existing_frame) {
+    GstAV1Picture *ref_picture;
+
+    ref_picture = priv->dpb->pic_list[frame_header->frame_to_show_map_idx];
+    if (!ref_picture) {
+      GST_WARNING_OBJECT (self, "Failed to find the frame index %d to show.",
+          frame_header->frame_to_show_map_idx);
+      return FALSE;
+    }
+
+    if (gst_av1_parser_reference_frame_loading (priv->parser,
+            &ref_picture->frame_hdr) != GST_AV1_PARSER_OK) {
+      GST_WARNING_OBJECT (self, "load the reference frame failed");
+      return FALSE;
+    }
+
+    g_assert (klass->duplicate_picture);
+    picture = klass->duplicate_picture (self, ref_picture);
+    if (!picture) {
+      GST_ERROR_OBJECT (self, "subclass didn't provide duplicated picture");
+      return FALSE;
+    }
+
+    picture->system_frame_number = priv->current_frame->system_frame_number;
+    picture->frame_hdr = *frame_header;
+    priv->current_picture = picture;
+  } else {
+    picture = gst_av1_picture_new ();
+    picture->frame_hdr = *frame_header;
+    picture->display_frame_id = frame_header->display_frame_id;
+    picture->show_frame = frame_header->show_frame;
+    picture->showable_frame = frame_header->showable_frame;
+    picture->apply_grain = frame_header->film_grain_params.apply_grain;
+    picture->system_frame_number = priv->current_frame->system_frame_number;
+
+    if (!frame_header->show_frame && !frame_header->showable_frame)
+      GST_VIDEO_CODEC_FRAME_FLAG_SET (priv->current_frame,
+          GST_VIDEO_CODEC_FRAME_FLAG_DECODE_ONLY);
+
+    if (klass->new_picture) {
+      if (!klass->new_picture (self, priv->current_frame, picture)) {
+        GST_ERROR_OBJECT (self, "new picture error");
+        return FALSE;
+      }
+    }
+    priv->current_picture = picture;
+
+    if (klass->start_picture) {
+      if (!klass->start_picture (self, picture, priv->dpb)) {
+        GST_ERROR_OBJECT (self, "start picture error");
+        return FALSE;
+      }
+    }
+  }
+
+  g_assert (priv->current_picture != NULL);
+
+  return TRUE;
+}
+
+static gboolean
+gst_av1_decoder_process_frame_header (GstAV1Decoder * self, GstAV1OBU * obu)
+{
+  GstAV1ParserResult res;
+  GstAV1DecoderPrivate *priv = self->priv;
+  GstAV1FrameHeaderOBU frame_header;
+
+  res = gst_av1_parser_parse_frame_header_obu (priv->parser, obu,
+      &frame_header);
+  if (res != GST_AV1_PARSER_OK) {
+    GST_WARNING_OBJECT (self, "Parsing frame header failed.");
+    return FALSE;
+  }
+
+  return gst_av1_decoder_decode_frame_header (self, &frame_header);
+}
+
+static gboolean
+gst_av1_decoder_process_tile_group (GstAV1Decoder * self, GstAV1OBU * obu)
+{
+  GstAV1ParserResult res;
+  GstAV1DecoderPrivate *priv = self->priv;
+  GstAV1TileGroupOBU tile_group;
+
+  res = gst_av1_parser_parse_tile_group_obu (priv->parser, obu, &tile_group);
+  if (res != GST_AV1_PARSER_OK) {
+    GST_WARNING_OBJECT (self, "Parsing tile group failed.");
+    return FALSE;
+  }
+
+  return gst_av1_decoder_decode_tile_group (self, &tile_group, obu);
+}
+
+static gboolean
+gst_av1_decoder_process_frame (GstAV1Decoder * self, GstAV1OBU * obu)
+{
+  GstAV1ParserResult res;
+  GstAV1DecoderPrivate *priv = self->priv;
+  GstAV1FrameOBU frame;
+
+  res = gst_av1_parser_parse_frame_obu (priv->parser, obu, &frame);
+  if (res != GST_AV1_PARSER_OK) {
+    GST_WARNING_OBJECT (self, "Parsing frame failed.");
+    return FALSE;
+  }
+
+  return gst_av1_decoder_decode_frame_header (self, &frame.frame_header) &&
+      gst_av1_decoder_decode_tile_group (self, &frame.tile_group, obu);
+}
+
+static gboolean
+gst_av1_decoder_temporal_delimiter (GstAV1Decoder * self, GstAV1OBU * obu)
+{
+  GstAV1DecoderPrivate *priv = self->priv;
+
+  return gst_av1_parser_parse_temporal_delimiter_obu (priv->parser,
+      obu) == GST_AV1_PARSER_OK;
+}
+
+static gboolean
+gst_av1_decoder_decode_one_obu (GstAV1Decoder * self, GstAV1OBU * obu)
+{
+  gboolean ret = TRUE;
+
+  GST_LOG_OBJECT (self, "Decode obu %s", get_obu_name (obu->obu_type));
+  switch (obu->obu_type) {
+    case GST_AV1_OBU_SEQUENCE_HEADER:
+      ret = gst_av1_decoder_process_sequence (self, obu);
+      break;
+    case GST_AV1_OBU_FRAME_HEADER:
+      ret = gst_av1_decoder_process_frame_header (self, obu);
+      break;
+    case GST_AV1_OBU_FRAME:
+      ret = gst_av1_decoder_process_frame (self, obu);
+      break;
+    case GST_AV1_OBU_TILE_GROUP:
+      ret = gst_av1_decoder_process_tile_group (self, obu);
+      break;
+    case GST_AV1_OBU_TEMPORAL_DELIMITER:
+      ret = gst_av1_decoder_temporal_delimiter (self, obu);
+      break;
+      /* TODO: may need to handled. */
+    case GST_AV1_OBU_METADATA:
+    case GST_AV1_OBU_REDUNDANT_FRAME_HEADER:
+    case GST_AV1_OBU_TILE_LIST:
+    case GST_AV1_OBU_PADDING:
+      ret = TRUE;
+      break;
+    default:
+      GST_WARNING_OBJECT (self, "an unrecognized obu type %d", obu->obu_type);
+      ret = FALSE;
+      break;
+  }
+
+  if (!ret)
+    GST_WARNING_OBJECT (self, "Failed to handle %s OBU",
+        get_obu_name (obu->obu_type));
+
+  return ret;
+}
+
+static void
+gst_av1_decoder_update_state (GstAV1Decoder * self)
+{
+  GstAV1DecoderPrivate *priv = self->priv;
+  GstAV1Picture *picture = priv->current_picture;
+  GstAV1ParserResult res;
+  GstAV1FrameHeaderOBU *fh;
+
+  g_assert (picture);
+  fh = &picture->frame_hdr;
+
+  /* This is a show_existing_frame case, only update key frame */
+  if (fh->show_existing_frame && fh->frame_type != GST_AV1_KEY_FRAME)
+    return;
+
+  res = gst_av1_parser_reference_frame_update (priv->parser, fh);
+  if (res != GST_AV1_PARSER_OK) {
+    GST_ERROR_OBJECT (self, "failed to update the reference.");
+    return;
+  }
+
+  gst_av1_dpb_add (priv->dpb, gst_av1_picture_ref (picture));
+}
+
+static GstFlowReturn
+gst_av1_decoder_handle_frame (GstVideoDecoder * decoder,
+    GstVideoCodecFrame * frame)
+{
+  GstAV1Decoder *self = GST_AV1_DECODER (decoder);
+  GstAV1DecoderPrivate *priv = self->priv;
+  GstAV1DecoderClass *klass = GST_AV1_DECODER_GET_CLASS (self);
+  GstBuffer *in_buf = frame->input_buffer;
+  GstMapInfo map;
+  GstFlowReturn ret = GST_FLOW_OK;
+  guint32 total_consumed, consumed;
+  GstAV1OBU obu;
+  GstAV1ParserResult res;
+
+  GST_LOG_OBJECT (self, "handle frame id %d, buf %" GST_PTR_FORMAT,
+      frame->system_frame_number, in_buf);
+
+  priv->current_frame = frame;
+  g_assert (!priv->current_picture);
+
+  if (!gst_buffer_map (in_buf, &map, GST_MAP_READ)) {
+    priv->current_frame = NULL;
+    GST_ERROR_OBJECT (self, "can not map input buffer");
+    ret = GST_FLOW_ERROR;
+    return ret;
+  }
+
+  total_consumed = 0;
+  while (total_consumed < map.size) {
+    res = gst_av1_parser_identify_one_obu (priv->parser,
+        map.data + total_consumed, map.size, &obu, &consumed);
+    if (res != GST_AV1_PARSER_OK) {
+      ret = GST_FLOW_ERROR;
+      goto out;
+    }
+
+    if (!gst_av1_decoder_decode_one_obu (self, &obu)) {
+      ret = GST_FLOW_ERROR;
+      goto out;
+    }
+
+    total_consumed += consumed;
+  }
+
+  if (!priv->current_picture) {
+    GST_ERROR_OBJECT (self, "No valid picture after exhaust input frame");
+    ret = GST_FLOW_ERROR;
+    goto out;
+  }
+
+  if (!priv->current_picture->frame_hdr.show_existing_frame) {
+    if (klass->end_picture) {
+      if (!klass->end_picture (self, priv->current_picture)) {
+        ret = GST_FLOW_ERROR;
+        GST_ERROR_OBJECT (self, "end picture error");
+        goto out;
+      }
+    }
+  }
+
+  gst_av1_decoder_update_state (self);
+
+out:
+  gst_buffer_unmap (in_buf, &map);
+
+  if (ret == GST_FLOW_OK) {
+    if (priv->current_picture->frame_hdr.show_frame ||
+        priv->current_picture->frame_hdr.show_existing_frame) {
+      g_assert (klass->output_picture);
+      /* transfer ownership of frame and picture */
+      ret = klass->output_picture (self, frame, priv->current_picture);
+    } else {
+      GST_LOG_OBJECT (self, "Decode only picture %p", priv->current_picture);
+      GST_VIDEO_CODEC_FRAME_SET_DECODE_ONLY (frame);
+      gst_av1_picture_unref (priv->current_picture);
+      ret = gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame);
+    }
+  } else {
+    if (priv->current_picture)
+      gst_av1_picture_unref (priv->current_picture);
+
+    GST_VIDEO_DECODER_ERROR (decoder, 1, STREAM, DECODE,
+        ("Failed to handle the frame %d", frame->system_frame_number),
+        NULL, ret);
+    gst_video_decoder_drop_frame (decoder, frame);
+  }
+
+  priv->current_picture = NULL;
+  priv->current_frame = NULL;
+  return ret;
+}
diff --git a/gst-libs/gst/codecs/gstav1decoder.h b/gst-libs/gst/codecs/gstav1decoder.h
new file mode 100644
index 000000000..4e5d0ebf6
--- /dev/null
+++ b/gst-libs/gst/codecs/gstav1decoder.h
@@ -0,0 +1,173 @@
+/* GStreamer
+ * Copyright (C) 2020 He Junyan <junyan.he@intel.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_AV1_DECODER_H__
+#define __GST_AV1_DECODER_H__
+
+#include <gst/codecs/codecs-prelude.h>
+
+#include <gst/video/video.h>
+#include <gst/codecparsers/gstav1parser.h>
+#include <gst/codecs/gstav1picture.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_AV1_DECODER            (gst_av1_decoder_get_type())
+#define GST_AV1_DECODER(obj)            (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_AV1_DECODER,GstAV1Decoder))
+#define GST_AV1_DECODER_CLASS(klass)    (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_AV1_DECODER,GstAV1DecoderClass))
+#define GST_AV1_DECODER_GET_CLASS(obj)  (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_AV1_DECODER,GstAV1DecoderClass))
+#define GST_IS_AV1_DECODER(obj)         (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_AV1_DECODER))
+#define GST_IS_AV1_DECODER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_AV1_DECODER))
+
+typedef struct _GstAV1Decoder GstAV1Decoder;
+typedef struct _GstAV1DecoderClass GstAV1DecoderClass;
+typedef struct _GstAV1DecoderPrivate GstAV1DecoderPrivate;
+
+/**
+ * GstAV1Decoder:
+ *
+ * The opaque #GstAV1Decoder data structure.
+ *
+ * Since: 1.20
+ */
+struct _GstAV1Decoder
+{
+  /*< private >*/
+  GstVideoDecoder parent;
+
+  /*< protected >*/
+  GstVideoCodecState * input_state;
+
+  /*< private >*/
+  GstAV1DecoderPrivate *priv;
+  gpointer padding[GST_PADDING_LARGE];
+};
+
+/**
+ * GstAV1DecoderClass:
+ */
+struct _GstAV1DecoderClass
+{
+  GstVideoDecoderClass parent_class;
+
+  /**
+   * GstAV1DecoderClass::new_sequence:
+   * @decoder: a #GstAV1Decoder
+   * @seq_hdr: a #GstAV1SequenceHeaderOBU
+   *
+   * Notifies subclass of SPS update
+   *
+   * Since: 1.20
+   */
+  gboolean        (*new_sequence)      (GstAV1Decoder * decoder,
+                                        const GstAV1SequenceHeaderOBU * seq_hdr);
+  /**
+   * GstAV1DecoderClass::new_picture:
+   * @decoder: a #GstAV1Decoder
+   * @frame: (transfer none): a #GstVideoCodecFrame
+   * @picture: (transfer none): a #GstAV1Picture
+   *
+   * Optional. Called whenever new #GstAV1Picture is created.
+   * Subclass can set implementation specific user data
+   * on the #GstAV1Picture via gst_av1_picture_set_user_data()
+   *
+   * Since: 1.20
+   */
+  gboolean        (*new_picture)       (GstAV1Decoder * decoder,
+                                        GstVideoCodecFrame * frame,
+                                        GstAV1Picture * picture);
+  /**
+   * GstAV1DecoderClass::duplicate_picture:
+   * @decoder: a #GstAV1Decoder
+   * @picture: (transfer none): a #GstAV1Picture
+   *
+   * Optional. Called when need to duplicate an existing
+   * #GstAV1Picture.
+   *
+   * Since: 1.20
+   */
+  GstAV1Picture * (*duplicate_picture) (GstAV1Decoder * decoder,
+                                        GstAV1Picture * picture);
+  /**
+   * GstAV1DecoderClass::start_picture:
+   * @decoder: a #GstAV1Decoder
+   * @picture: (transfer none): a #GstAV1Picture
+   * @dpb: (transfer none): a #GstAV1Dpb
+   *
+   * Optional. Called per one #GstAV1Picture to notify subclass to prepare
+   * decoding process for the #GstAV1Picture
+   *
+   * Since: 1.20
+   */
+  gboolean        (*start_picture)     (GstAV1Decoder * decoder,
+                                        GstAV1Picture * picture,
+                                        GstAV1Dpb * dpb);
+  /**
+   * GstAV1DecoderClass::decode_tile:
+   * @decoder: a #GstAV1Decoder
+   * @picture: (transfer none): a #GstAV1Picture
+   * @tile: (transfer none): a #GstAV1Tile
+   *
+   * Provides the tile data with tile group header and required raw
+   * bitstream for subclass to decode it.
+   *
+   * Since: 1.20
+   */
+  gboolean        (*decode_tile)       (GstAV1Decoder * decoder,
+                                        GstAV1Picture * picture,
+                                        GstAV1Tile * tile);
+  /**
+   * GstAV1DecoderClass::end_picture:
+   * @decoder: a #GstAV1Decoder
+   * @picture: (transfer none): a #GstAV1Picture
+   *
+   * Optional. Called per one #GstAV1Picture to notify subclass to finish
+   * decoding process for the #GstAV1Picture
+   *
+   * Since: 1.20
+   */
+  gboolean        (*end_picture)       (GstAV1Decoder * decoder,
+                                        GstAV1Picture * picture);
+  /**
+   * GstAV1DecoderClass::output_picture:
+   * @decoder: a #GstAV1Decoder
+   * @frame: (transfer full): a #GstVideoCodecFrame
+   * @picture: (transfer full): a #GstAV1Picture
+   *
+   * Called with a #GstAV1Picture which is required to be outputted.
+   * The #GstVideoCodecFrame must be consumed by subclass.
+   *
+   * Since: 1.20
+   */
+  GstFlowReturn   (*output_picture)    (GstAV1Decoder * decoder,
+                                        GstVideoCodecFrame * frame,
+                                        GstAV1Picture * picture);
+
+  /*< private >*/
+  gpointer padding[GST_PADDING_LARGE];
+};
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstAV1Decoder, gst_object_unref)
+
+GST_CODECS_API
+GType gst_av1_decoder_get_type (void);
+
+G_END_DECLS
+
+#endif /* __GST_AV1_DECODER_H__ */
diff --git a/gst-libs/gst/codecs/gstav1picture.c b/gst-libs/gst/codecs/gstav1picture.c
new file mode 100644
index 000000000..c2fa02268
--- /dev/null
+++ b/gst-libs/gst/codecs/gstav1picture.c
@@ -0,0 +1,194 @@
+/* GStreamer
+ * Copyright (C) 2020 He Junyan <junyan.he@intel.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "gstav1picture.h"
+
+GST_DEBUG_CATEGORY_EXTERN (gst_av1_decoder_debug);
+#define GST_CAT_DEFAULT gst_av1_decoder_debug
+
+GST_DEFINE_MINI_OBJECT_TYPE (GstAV1Picture, gst_av1_picture);
+
+static void
+_gst_av1_picture_free (GstAV1Picture * picture)
+{
+  GST_TRACE ("Free picture %p", picture);
+
+  if (picture->notify)
+    picture->notify (picture->user_data);
+
+  g_free (picture);
+}
+
+/**
+ * gst_av1_picture_new:
+ *
+ * Create new #GstAV1Picture
+ *
+ * Returns: a new #GstAV1Picture
+ *
+ * Since: 1.20
+ */
+GstAV1Picture *
+gst_av1_picture_new (void)
+{
+  GstAV1Picture *pic;
+
+  pic = g_new0 (GstAV1Picture, 1);
+
+  gst_mini_object_init (GST_MINI_OBJECT_CAST (pic), 0,
+      GST_TYPE_AV1_PICTURE, NULL, NULL,
+      (GstMiniObjectFreeFunction) _gst_av1_picture_free);
+
+  GST_TRACE ("New picture %p", pic);
+
+  return pic;
+}
+
+/**
+ * gst_av1_picture_set_user_data:
+ * @picture: a #GstAV1Picture
+ * @user_data: private data
+ * @notify: (closure user_data): a #GDestroyNotify
+ *
+ * Sets @user_data on the picture and the #GDestroyNotify that will be called when
+ * the picture is freed.
+ *
+ * If a @user_data was previously set, then the previous set @notify will be called
+ * before the @user_data is replaced.
+ *
+ * Since: 1.20
+ */
+void
+gst_av1_picture_set_user_data (GstAV1Picture * picture, gpointer user_data,
+    GDestroyNotify notify)
+{
+  g_return_if_fail (GST_IS_AV1_PICTURE (picture));
+
+  if (picture->notify)
+    picture->notify (picture->user_data);
+
+  picture->user_data = user_data;
+  picture->notify = notify;
+}
+
+/**
+ * gst_av1_picture_get_user_data:
+ * @picture: a #GstAV1Picture
+ *
+ * Gets private data set on the picture via
+ * gst_av1_picture_set_user_data() previously.
+ *
+ * Returns: (transfer none): The previously set user_data
+ *
+ * Since: 1.20
+ */
+gpointer
+gst_av1_picture_get_user_data (GstAV1Picture * picture)
+{
+  return picture->user_data;
+}
+
+/**
+ * gst_av1_dpb_new: (skip)
+ *
+ * Create new #GstAV1Dpb
+ *
+ * Returns: a new #GstAV1Dpb
+ *
+ * Since: 1.20
+ */
+GstAV1Dpb *
+gst_av1_dpb_new (void)
+{
+  GstAV1Dpb *dpb;
+
+  dpb = g_new0 (GstAV1Dpb, 1);
+
+  return dpb;
+}
+
+/**
+ * gst_av1_dpb_free:
+ * @dpb: a #GstAV1Dpb to free
+ *
+ * Free the @dpb
+ *
+ * Since: 1.20
+ */
+void
+gst_av1_dpb_free (GstAV1Dpb * dpb)
+{
+  g_return_if_fail (dpb != NULL);
+
+  gst_av1_dpb_clear (dpb);
+  g_free (dpb);
+}
+
+/**
+ * gst_av1_dpb_clear:
+ * @dpb: a #GstAV1Dpb
+ *
+ * Clear all stored #GstAV1Picture
+ *
+ * Since: 1.20
+ */
+void
+gst_av1_dpb_clear (GstAV1Dpb * dpb)
+{
+  gint i;
+
+  g_return_if_fail (dpb != NULL);
+
+  for (i = 0; i < GST_AV1_NUM_REF_FRAMES; i++)
+    gst_av1_picture_clear (&dpb->pic_list[i]);
+}
+
+/**
+ * gst_av1_dpb_add:
+ * @dpb: a #GstAV1Dpb
+ * @picture: (transfer full): a #GstAV1Picture
+ *
+ * Store the @picture
+ *
+ * Since: 1.20
+ */
+void
+gst_av1_dpb_add (GstAV1Dpb * dpb, GstAV1Picture * picture)
+{
+  GstAV1FrameHeaderOBU *fh;
+  guint i;
+
+  g_return_if_fail (dpb != NULL);
+  g_return_if_fail (GST_IS_AV1_PICTURE (picture));
+
+  fh = &picture->frame_hdr;
+
+  for (i = 0; i < GST_AV1_NUM_REF_FRAMES; i++) {
+    if ((fh->refresh_frame_flags >> i) & 1) {
+      GST_TRACE ("reference frame %p to ref slot:%d", picture, i);
+      gst_av1_picture_replace (&dpb->pic_list[i], picture);
+    }
+  }
+
+  gst_av1_picture_unref (picture);
+}
diff --git a/gst-libs/gst/codecs/gstav1picture.h b/gst-libs/gst/codecs/gstav1picture.h
new file mode 100644
index 000000000..916f97fdb
--- /dev/null
+++ b/gst-libs/gst/codecs/gstav1picture.h
@@ -0,0 +1,161 @@
+/* GStreamer
+ * Copyright (C) 2020 He Junyan <junyan.he@intel.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_AV1_PICTURE_H__
+#define __GST_AV1_PICTURE_H__
+
+#include <gst/codecs/codecs-prelude.h>
+#include <gst/codecparsers/gstav1parser.h>
+
+G_BEGIN_DECLS
+
+/**
+ * GST_TYPE_AV1_PICTURE:
+ *
+ * Since: 1.20
+ */
+#define GST_TYPE_AV1_PICTURE     (gst_av1_picture_get_type())
+/**
+ * GST_IS_AV1_PICTURE:
+ *
+ * Since: 1.20
+ */
+#define GST_IS_AV1_PICTURE(obj)  (GST_IS_MINI_OBJECT_TYPE(obj, GST_TYPE_AV1_PICTURE))
+/**
+ * GST_AV1_PICTURE:
+ *
+ * Since: 1.20
+ */
+#define GST_AV1_PICTURE(obj)     ((GstAV1Picture *)obj)
+
+typedef struct _GstAV1Picture GstAV1Picture;
+typedef struct _GstAV1Tile GstAV1Tile;
+
+/**
+ * GstAV1Tile:
+ *
+ * Since: 1.20
+ */
+struct _GstAV1Tile
+{
+  GstAV1TileGroupOBU tile_group;
+  /* raw data and size of tile group (does not have ownership) */
+  GstAV1OBU obu;
+};
+
+/**
+ * GstAV1Picture:
+ *
+ * Since: 1.20
+ */
+struct _GstAV1Picture
+{
+  GstMiniObject parent;
+
+  /* From GstVideoCodecFrame */
+  guint32 system_frame_number;
+
+  GstAV1FrameHeaderOBU frame_hdr;
+
+  /* copied from parser */
+  guint32 display_frame_id;
+  gboolean show_frame;
+  gboolean showable_frame;
+  gboolean apply_grain;
+
+  gpointer user_data;
+  GDestroyNotify notify;
+};
+
+GST_CODECS_API
+GType gst_av1_picture_get_type (void);
+
+GST_CODECS_API
+GstAV1Picture * gst_av1_picture_new (void);
+
+static inline GstAV1Picture *
+gst_av1_picture_ref (GstAV1Picture * picture)
+{
+  return (GstAV1Picture *) gst_mini_object_ref (GST_MINI_OBJECT_CAST (picture));
+}
+
+static inline void
+gst_av1_picture_unref (GstAV1Picture * picture)
+{
+  gst_mini_object_unref (GST_MINI_OBJECT_CAST (picture));
+}
+
+static inline gboolean
+gst_av1_picture_replace (GstAV1Picture ** old_picture,
+    GstAV1Picture * new_picture)
+{
+  return gst_mini_object_replace ((GstMiniObject **) old_picture,
+      (GstMiniObject *) new_picture);
+}
+
+static inline void
+gst_av1_picture_clear (GstAV1Picture ** picture)
+{
+  if (picture && *picture) {
+    gst_av1_picture_unref (*picture);
+    *picture = NULL;
+  }
+}
+
+GST_CODECS_API
+void gst_av1_picture_set_user_data (GstAV1Picture * picture,
+                                    gpointer user_data,
+                                    GDestroyNotify notify);
+
+GST_CODECS_API
+gpointer gst_av1_picture_get_user_data (GstAV1Picture * picture);
+
+/*******************
+ * GstAV1Dpb *
+ *******************/
+typedef struct _GstAV1Dpb GstAV1Dpb;
+
+/**
+ * GstAV1Dpb:
+ *
+ * Since: 1.20
+ */
+struct _GstAV1Dpb
+{
+  GstAV1Picture *pic_list[GST_AV1_NUM_REF_FRAMES];
+};
+
+GST_CODECS_API
+GstAV1Dpb * gst_av1_dpb_new (void);
+
+GST_CODECS_API
+void  gst_av1_dpb_free             (GstAV1Dpb * dpb);
+
+GST_CODECS_API
+void  gst_av1_dpb_clear            (GstAV1Dpb * dpb);
+
+GST_CODECS_API
+void  gst_av1_dpb_add              (GstAV1Dpb * dpb,
+                                    GstAV1Picture * picture);
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstAV1Picture, gst_av1_picture_unref)
+
+G_END_DECLS
+
+#endif /* __GST_AV1_PICTURE_H__ */
diff --git a/gst-libs/gst/codecs/meson.build b/gst-libs/gst/codecs/meson.build
index f8cc502c4..d77241e7e 100644
--- a/gst-libs/gst/codecs/meson.build
+++ b/gst-libs/gst/codecs/meson.build
@@ -9,6 +9,8 @@ codecs_sources = files([
   'gstvp8picture.c',
   'gstmpeg2decoder.c',
   'gstmpeg2picture.c',
+  'gstav1decoder.c',
+  'gstav1picture.c',
 ])
 
 codecs_headers = [
@@ -22,6 +24,8 @@ codecs_headers = [
   'gstvp8picture.h',
   'gstmpeg2decoder.h',
   'gstmpeg2picture.h',
+  'gstav1decoder.h',
+  'gstav1picture.h',
 ]
 
 cp_args = [