/* GStreamer * Copyright (C) 2020 He Junyan * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ /** * SECTION:gstav1decoder * @title: Gstav1Decoder * @short_description: Base class to implement stateless AV1 decoders * @sources: * - gstav1picture.h */ #ifdef HAVE_CONFIG_H #include #endif #include "gstav1decoder.h" GST_DEBUG_CATEGORY (gst_av1_decoder_debug); #define GST_CAT_DEFAULT gst_av1_decoder_debug struct _GstAV1DecoderPrivate { gint max_width; gint max_height; GstAV1Profile profile; GstAV1Parser *parser; GstAV1Dpb *dpb; GstAV1Picture *current_picture; GstVideoCodecFrame *current_frame; }; #define parent_class gst_av1_decoder_parent_class G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstAV1Decoder, gst_av1_decoder, GST_TYPE_VIDEO_DECODER, G_ADD_PRIVATE (GstAV1Decoder); GST_DEBUG_CATEGORY_INIT (gst_av1_decoder_debug, "av1decoder", 0, "AV1 Video Decoder")); static gint _floor_log2 (guint32 x) { gint s = 0; while (x != 0) { x = x >> 1; s++; } return s - 1; } static gboolean gst_av1_decoder_start (GstVideoDecoder * decoder); static gboolean gst_av1_decoder_stop (GstVideoDecoder * decoder); static gboolean gst_av1_decoder_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state); static GstFlowReturn gst_av1_decoder_finish (GstVideoDecoder * decoder); static gboolean gst_av1_decoder_flush (GstVideoDecoder * decoder); static GstFlowReturn gst_av1_decoder_drain (GstVideoDecoder * decoder); static GstFlowReturn gst_av1_decoder_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame); static GstAV1Picture *gst_av1_decoder_duplicate_picture_default (GstAV1Decoder * decoder, GstAV1Picture * picture); static void gst_av1_decoder_class_init (GstAV1DecoderClass * klass) { GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass); decoder_class->start = GST_DEBUG_FUNCPTR (gst_av1_decoder_start); decoder_class->stop = GST_DEBUG_FUNCPTR (gst_av1_decoder_stop); decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_av1_decoder_set_format); decoder_class->finish = GST_DEBUG_FUNCPTR (gst_av1_decoder_finish); decoder_class->flush = GST_DEBUG_FUNCPTR (gst_av1_decoder_flush); decoder_class->drain = GST_DEBUG_FUNCPTR (gst_av1_decoder_drain); decoder_class->handle_frame = GST_DEBUG_FUNCPTR (gst_av1_decoder_handle_frame); klass->duplicate_picture = GST_DEBUG_FUNCPTR (gst_av1_decoder_duplicate_picture_default); } static void gst_av1_decoder_init (GstAV1Decoder * self) { gst_video_decoder_set_packetized (GST_VIDEO_DECODER (self), TRUE); self->priv = gst_av1_decoder_get_instance_private (self); } static void gst_av1_decoder_reset (GstAV1Decoder * self) { GstAV1DecoderPrivate *priv = self->priv; priv->max_width = 0; priv->max_height = 0; gst_av1_picture_clear (&priv->current_picture); priv->current_frame = NULL; priv->profile = GST_AV1_PROFILE_UNDEFINED; if (priv->dpb) gst_av1_dpb_clear (priv->dpb); if (priv->parser) gst_av1_parser_reset (priv->parser, FALSE); } static gboolean gst_av1_decoder_start (GstVideoDecoder * decoder) { GstAV1Decoder *self = GST_AV1_DECODER (decoder); GstAV1DecoderPrivate *priv = self->priv; priv->parser = gst_av1_parser_new (); priv->dpb = gst_av1_dpb_new (); gst_av1_decoder_reset (self); return TRUE; } static gboolean gst_av1_decoder_stop (GstVideoDecoder * decoder) { GstAV1Decoder *self = GST_AV1_DECODER (decoder); GstAV1DecoderPrivate *priv = self->priv; gst_av1_decoder_reset (self); g_clear_pointer (&self->input_state, gst_video_codec_state_unref); g_clear_pointer (&priv->parser, gst_av1_parser_free); g_clear_pointer (&priv->dpb, gst_av1_dpb_free); return TRUE; } static gboolean gst_av1_decoder_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) { GstAV1Decoder *self = GST_AV1_DECODER (decoder); GstAV1DecoderPrivate *priv = self->priv; GST_DEBUG_OBJECT (decoder, "Set format"); if (self->input_state) gst_video_codec_state_unref (self->input_state); self->input_state = gst_video_codec_state_ref (state); priv->max_width = GST_VIDEO_INFO_WIDTH (&state->info); priv->max_height = GST_VIDEO_INFO_HEIGHT (&state->info); return TRUE; } static GstFlowReturn gst_av1_decoder_finish (GstVideoDecoder * decoder) { GST_DEBUG_OBJECT (decoder, "finish"); gst_av1_decoder_reset (GST_AV1_DECODER (decoder)); return GST_FLOW_OK; } static gboolean gst_av1_decoder_flush (GstVideoDecoder * decoder) { GST_DEBUG_OBJECT (decoder, "flush"); gst_av1_decoder_reset (GST_AV1_DECODER (decoder)); return TRUE; } static GstFlowReturn gst_av1_decoder_drain (GstVideoDecoder * decoder) { GST_DEBUG_OBJECT (decoder, "drain"); gst_av1_decoder_reset (GST_AV1_DECODER (decoder)); return GST_FLOW_OK; } static GstAV1Picture * gst_av1_decoder_duplicate_picture_default (GstAV1Decoder * decoder, GstAV1Picture * picture) { GstAV1Picture *new_picture; new_picture = gst_av1_picture_new (); return new_picture; } static const gchar * get_obu_name (GstAV1OBUType type) { switch (type) { case GST_AV1_OBU_SEQUENCE_HEADER: return "sequence header"; case GST_AV1_OBU_TEMPORAL_DELIMITER: return "temporal delimiter"; case GST_AV1_OBU_FRAME_HEADER: return "frame header"; case GST_AV1_OBU_TILE_GROUP: return "tile group"; case GST_AV1_OBU_METADATA: return "metadata"; case GST_AV1_OBU_FRAME: return "frame"; case GST_AV1_OBU_REDUNDANT_FRAME_HEADER: return "redundant frame header"; case GST_AV1_OBU_TILE_LIST: return "tile list"; case GST_AV1_OBU_PADDING: return "padding"; default: return "unknown"; } return NULL; } static const gchar * gst_av1_decoder_profile_to_string (GstAV1Profile profile) { switch (profile) { case GST_AV1_PROFILE_0: return "0"; case GST_AV1_PROFILE_1: return "1"; case GST_AV1_PROFILE_2: return "2"; default: break; } return NULL; } static GstFlowReturn gst_av1_decoder_process_sequence (GstAV1Decoder * self, GstAV1OBU * obu) { GstAV1ParserResult res; GstAV1DecoderPrivate *priv = self->priv; GstAV1SequenceHeaderOBU seq_header; GstAV1SequenceHeaderOBU old_seq_header = { 0, }; GstAV1DecoderClass *klass = GST_AV1_DECODER_GET_CLASS (self); GstFlowReturn ret = GST_FLOW_OK; if (priv->parser->seq_header) old_seq_header = *priv->parser->seq_header; res = gst_av1_parser_parse_sequence_header_obu (priv->parser, obu, &seq_header); if (res != GST_AV1_PARSER_OK) { GST_WARNING_OBJECT (self, "Parsing sequence failed."); return GST_FLOW_ERROR; } if (!memcmp (&old_seq_header, &seq_header, sizeof (GstAV1SequenceHeaderOBU))) { GST_DEBUG_OBJECT (self, "Get same sequence header."); return GST_FLOW_OK; } g_assert (klass->new_sequence); GST_DEBUG_OBJECT (self, "Sequence updated, profile %s -> %s, max resolution: %dx%d -> %dx%d", gst_av1_decoder_profile_to_string (priv->profile), gst_av1_decoder_profile_to_string (seq_header.seq_profile), priv->max_width, priv->max_height, seq_header.max_frame_width_minus_1 + 1, seq_header.max_frame_height_minus_1 + 1); ret = klass->new_sequence (self, &seq_header); if (ret != GST_FLOW_OK) { GST_ERROR_OBJECT (self, "subclass does not want accept new sequence"); return ret; } priv->profile = seq_header.seq_profile; priv->max_width = seq_header.max_frame_width_minus_1 + 1; priv->max_height = seq_header.max_frame_height_minus_1 + 1; gst_av1_dpb_clear (priv->dpb); return GST_FLOW_OK; } static GstFlowReturn gst_av1_decoder_decode_tile_group (GstAV1Decoder * self, GstAV1TileGroupOBU * tile_group, GstAV1OBU * obu) { GstAV1DecoderPrivate *priv = self->priv; GstAV1DecoderClass *klass = GST_AV1_DECODER_GET_CLASS (self); GstAV1Picture *picture = priv->current_picture; GstAV1Tile tile; GstFlowReturn ret = GST_FLOW_OK; if (!picture) { GST_ERROR_OBJECT (self, "No picture has created for current frame"); return GST_FLOW_ERROR; } if (picture->frame_hdr.show_existing_frame) { GST_ERROR_OBJECT (self, "Current picture is showing the existing frame."); return GST_FLOW_ERROR; } tile.obu = *obu; tile.tile_group = *tile_group; g_assert (klass->decode_tile); ret = klass->decode_tile (self, picture, &tile); if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT (self, "Decode tile error"); return ret; } return GST_FLOW_OK; } static GstFlowReturn gst_av1_decoder_decode_frame_header (GstAV1Decoder * self, GstAV1FrameHeaderOBU * frame_header) { GstAV1DecoderPrivate *priv = self->priv; GstAV1DecoderClass *klass = GST_AV1_DECODER_GET_CLASS (self); GstAV1Picture *picture = NULL; GstFlowReturn ret = GST_FLOW_OK; g_assert (priv->current_frame); if (priv->current_picture != NULL) { GST_ERROR_OBJECT (self, "Already have picture for current frame"); return GST_FLOW_ERROR; } if (frame_header->show_existing_frame) { GstAV1Picture *ref_picture; ref_picture = priv->dpb->pic_list[frame_header->frame_to_show_map_idx]; if (!ref_picture) { GST_WARNING_OBJECT (self, "Failed to find the frame index %d to show.", frame_header->frame_to_show_map_idx); return GST_FLOW_ERROR; } if (gst_av1_parser_reference_frame_loading (priv->parser, &ref_picture->frame_hdr) != GST_AV1_PARSER_OK) { GST_WARNING_OBJECT (self, "load the reference frame failed"); return GST_FLOW_ERROR; } /* FIXME: duplicate picture might be optional feature like that of VP9 * decoder baseclass */ g_assert (klass->duplicate_picture); picture = klass->duplicate_picture (self, ref_picture); if (!picture) { GST_ERROR_OBJECT (self, "subclass didn't provide duplicated picture"); return GST_FLOW_ERROR; } picture->system_frame_number = priv->current_frame->system_frame_number; picture->frame_hdr = *frame_header; picture->frame_hdr.render_width = ref_picture->frame_hdr.render_width; picture->frame_hdr.render_height = ref_picture->frame_hdr.render_height; priv->current_picture = picture; } else { picture = gst_av1_picture_new (); picture->frame_hdr = *frame_header; picture->display_frame_id = frame_header->display_frame_id; picture->show_frame = frame_header->show_frame; picture->showable_frame = frame_header->showable_frame; picture->apply_grain = frame_header->film_grain_params.apply_grain; picture->system_frame_number = priv->current_frame->system_frame_number; if (!frame_header->show_frame && !frame_header->showable_frame) GST_VIDEO_CODEC_FRAME_FLAG_SET (priv->current_frame, GST_VIDEO_CODEC_FRAME_FLAG_DECODE_ONLY); if (klass->new_picture) { ret = klass->new_picture (self, priv->current_frame, picture); if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT (self, "new picture error"); return ret; } } priv->current_picture = picture; if (klass->start_picture) { ret = klass->start_picture (self, picture, priv->dpb); if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT (self, "start picture error"); return ret; } } } g_assert (priv->current_picture != NULL); return GST_FLOW_OK; } static GstFlowReturn gst_av1_decoder_process_frame_header (GstAV1Decoder * self, GstAV1OBU * obu) { GstAV1ParserResult res; GstAV1DecoderPrivate *priv = self->priv; GstAV1FrameHeaderOBU frame_header; res = gst_av1_parser_parse_frame_header_obu (priv->parser, obu, &frame_header); if (res != GST_AV1_PARSER_OK) { GST_WARNING_OBJECT (self, "Parsing frame header failed."); return GST_FLOW_ERROR; } return gst_av1_decoder_decode_frame_header (self, &frame_header); } static GstFlowReturn gst_av1_decoder_process_tile_group (GstAV1Decoder * self, GstAV1OBU * obu) { GstAV1ParserResult res; GstAV1DecoderPrivate *priv = self->priv; GstAV1TileGroupOBU tile_group; res = gst_av1_parser_parse_tile_group_obu (priv->parser, obu, &tile_group); if (res != GST_AV1_PARSER_OK) { GST_WARNING_OBJECT (self, "Parsing tile group failed."); return GST_FLOW_ERROR; } return gst_av1_decoder_decode_tile_group (self, &tile_group, obu); } static GstFlowReturn gst_av1_decoder_process_frame (GstAV1Decoder * self, GstAV1OBU * obu) { GstAV1ParserResult res; GstAV1DecoderPrivate *priv = self->priv; GstAV1FrameOBU frame; GstFlowReturn ret = GST_FLOW_OK; res = gst_av1_parser_parse_frame_obu (priv->parser, obu, &frame); if (res != GST_AV1_PARSER_OK) { GST_WARNING_OBJECT (self, "Parsing frame failed."); return GST_FLOW_ERROR; } ret = gst_av1_decoder_decode_frame_header (self, &frame.frame_header); if (ret != GST_FLOW_OK) return ret; return gst_av1_decoder_decode_tile_group (self, &frame.tile_group, obu); } static GstFlowReturn gst_av1_decoder_temporal_delimiter (GstAV1Decoder * self, GstAV1OBU * obu) { GstAV1DecoderPrivate *priv = self->priv; if (gst_av1_parser_parse_temporal_delimiter_obu (priv->parser, obu) == GST_AV1_PARSER_OK) { return GST_FLOW_OK; } return GST_FLOW_ERROR; } static GstFlowReturn gst_av1_decoder_decode_one_obu (GstAV1Decoder * self, GstAV1OBU * obu) { GstFlowReturn ret = GST_FLOW_OK; GST_LOG_OBJECT (self, "Decode obu %s", get_obu_name (obu->obu_type)); switch (obu->obu_type) { case GST_AV1_OBU_SEQUENCE_HEADER: ret = gst_av1_decoder_process_sequence (self, obu); break; case GST_AV1_OBU_FRAME_HEADER: ret = gst_av1_decoder_process_frame_header (self, obu); break; case GST_AV1_OBU_FRAME: ret = gst_av1_decoder_process_frame (self, obu); break; case GST_AV1_OBU_TILE_GROUP: ret = gst_av1_decoder_process_tile_group (self, obu); break; case GST_AV1_OBU_TEMPORAL_DELIMITER: ret = gst_av1_decoder_temporal_delimiter (self, obu); break; /* TODO: may need to handled. */ case GST_AV1_OBU_METADATA: case GST_AV1_OBU_REDUNDANT_FRAME_HEADER: case GST_AV1_OBU_TILE_LIST: case GST_AV1_OBU_PADDING: break; default: GST_WARNING_OBJECT (self, "an unrecognized obu type %d", obu->obu_type); break; } if (ret != GST_FLOW_OK) GST_WARNING_OBJECT (self, "Failed to handle %s OBU", get_obu_name (obu->obu_type)); return ret; } static void gst_av1_decoder_update_state (GstAV1Decoder * self) { GstAV1DecoderPrivate *priv = self->priv; GstAV1Picture *picture = priv->current_picture; GstAV1ParserResult res; GstAV1FrameHeaderOBU *fh; g_assert (picture); fh = &picture->frame_hdr; /* This is a show_existing_frame case, only update key frame */ if (fh->show_existing_frame && fh->frame_type != GST_AV1_KEY_FRAME) return; res = gst_av1_parser_reference_frame_update (priv->parser, fh); if (res != GST_AV1_PARSER_OK) { GST_ERROR_OBJECT (self, "failed to update the reference."); return; } gst_av1_dpb_add (priv->dpb, gst_av1_picture_ref (picture)); } static GstFlowReturn gst_av1_decoder_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame) { GstAV1Decoder *self = GST_AV1_DECODER (decoder); GstAV1DecoderPrivate *priv = self->priv; GstAV1DecoderClass *klass = GST_AV1_DECODER_GET_CLASS (self); GstBuffer *in_buf = frame->input_buffer; GstMapInfo map; GstFlowReturn ret = GST_FLOW_OK; guint32 total_consumed, consumed; GstAV1OBU obu; GstAV1ParserResult res; GST_LOG_OBJECT (self, "handle frame id %d, buf %" GST_PTR_FORMAT, frame->system_frame_number, in_buf); priv->current_frame = frame; g_assert (!priv->current_picture); if (!gst_buffer_map (in_buf, &map, GST_MAP_READ)) { priv->current_frame = NULL; GST_ERROR_OBJECT (self, "can not map input buffer"); return GST_FLOW_ERROR; } total_consumed = 0; while (total_consumed < map.size) { res = gst_av1_parser_identify_one_obu (priv->parser, map.data + total_consumed, map.size, &obu, &consumed); if (res != GST_AV1_PARSER_OK) { ret = GST_FLOW_ERROR; goto out; } ret = gst_av1_decoder_decode_one_obu (self, &obu); if (ret != GST_FLOW_OK) { goto out; } total_consumed += consumed; } if (!priv->current_picture) { GST_ERROR_OBJECT (self, "No valid picture after exhaust input frame"); ret = GST_FLOW_ERROR; goto out; } if (!priv->current_picture->frame_hdr.show_existing_frame) { if (klass->end_picture) { ret = klass->end_picture (self, priv->current_picture); if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT (self, "end picture error"); goto out; } } } gst_av1_decoder_update_state (self); out: gst_buffer_unmap (in_buf, &map); if (ret == GST_FLOW_OK) { if (priv->current_picture->frame_hdr.show_frame || priv->current_picture->frame_hdr.show_existing_frame) { /* Only output one frame with the highest spatial id from each TU * when there are multiple spatial layers. */ if (priv->parser->state.operating_point_idc && obu.header.obu_spatial_id < _floor_log2 (priv->parser->state.operating_point_idc >> 8)) { gst_av1_picture_unref (priv->current_picture); gst_video_decoder_release_frame (decoder, frame); } else { g_assert (klass->output_picture); /* transfer ownership of frame and picture */ ret = klass->output_picture (self, frame, priv->current_picture); } } else { GST_LOG_OBJECT (self, "Decode only picture %p", priv->current_picture); GST_VIDEO_CODEC_FRAME_SET_DECODE_ONLY (frame); gst_av1_picture_unref (priv->current_picture); ret = gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame); } } else { if (priv->current_picture) gst_av1_picture_unref (priv->current_picture); gst_video_decoder_drop_frame (decoder, frame); } priv->current_picture = NULL; priv->current_frame = NULL; if (ret != GST_FLOW_OK) { GST_VIDEO_DECODER_ERROR (decoder, 1, STREAM, DECODE, ("Failed to handle the frame %d", frame->system_frame_number), NULL, ret); return ret; } return GST_FLOW_OK; }