codecs: vp8decoder: Use GstFlowReturn everywhere

boolean return value is not sufficient for representing the reason
of error in most cases. For instance, any errors around new_sequence()
would mean negotiation error, not just *ERROR*.
And some subclasses will allocate buffer/memory/surface on new_picture()
but it could be failed because of expected error, likely flushing

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2528>

4 files changed, 83 insertions, 66 deletions

diff --git a/sys/d3d11/gstd3d11vp8dec.cpp b/sys/d3d11/gstd3d11vp8dec.cpp
index 9e18c1217..a870a64a0 100644
--- a/sys/d3d11/gstd3d11vp8dec.cpp
+++ b/sys/d3d11/gstd3d11vp8dec.cpp
@@ -110,15 +110,15 @@ static gboolean gst_d3d11_vp8_sink_event (GstVideoDecoder * decoder,
     GstEvent * event);
 
 /* GstVp8Decoder */
-static gboolean gst_d3d11_vp8_dec_new_sequence (GstVp8Decoder * decoder,
+static GstFlowReturn gst_d3d11_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     const GstVp8FrameHdr * frame_hdr);
-static gboolean gst_d3d11_vp8_dec_new_picture (GstVp8Decoder * decoder,
+static GstFlowReturn gst_d3d11_vp8_dec_new_picture (GstVp8Decoder * decoder,
     GstVideoCodecFrame * frame, GstVp8Picture * picture);
-static gboolean gst_d3d11_vp8_dec_start_picture (GstVp8Decoder * decoder,
+static GstFlowReturn gst_d3d11_vp8_dec_start_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture);
-static gboolean gst_d3d11_vp8_dec_decode_picture (GstVp8Decoder * decoder,
+static GstFlowReturn gst_d3d11_vp8_dec_decode_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture, GstVp8Parser * parser);
-static gboolean gst_d3d11_vp8_dec_end_picture (GstVp8Decoder * decoder,
+static GstFlowReturn gst_d3d11_vp8_dec_end_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture);
 static GstFlowReturn gst_d3d11_vp8_dec_output_picture (GstVp8Decoder *
     decoder, GstVideoCodecFrame * frame, GstVp8Picture * picture);
@@ -312,7 +312,7 @@ gst_d3d11_vp8_sink_event (GstVideoDecoder * decoder, GstEvent * event)
   return GST_VIDEO_DECODER_CLASS (parent_class)->sink_event (decoder, event);
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     const GstVp8FrameHdr * frame_hdr)
 {
@@ -334,18 +334,18 @@ gst_d3d11_vp8_dec_new_sequence (GstVp8Decoder * decoder,
           decoder->input_state, &info, inner->width, inner->height,
           NUM_OUTPUT_VIEW)) {
     GST_ERROR_OBJECT (self, "Failed to create decoder");
-    return FALSE;
+    return GST_FLOW_NOT_NEGOTIATED;
   }
 
   if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
     GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-    return FALSE;
+    return GST_FLOW_NOT_NEGOTIATED;
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_vp8_dec_new_picture (GstVp8Decoder * decoder,
     GstVideoCodecFrame * frame, GstVp8Picture * picture)
 {
@@ -357,7 +357,7 @@ gst_d3d11_vp8_dec_new_picture (GstVp8Decoder * decoder,
       GST_VIDEO_DECODER (decoder));
   if (!view_buffer) {
     GST_DEBUG_OBJECT (self, "No available output view buffer");
-    return FALSE;
+    return GST_FLOW_FLUSHING;
   }
 
   GST_LOG_OBJECT (self, "New output view buffer %" GST_PTR_FORMAT, view_buffer);
@@ -367,10 +367,10 @@ gst_d3d11_vp8_dec_new_picture (GstVp8Decoder * decoder,
 
   GST_LOG_OBJECT (self, "New VP8 picture %p", picture);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_vp8_dec_start_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture)
 {
@@ -379,7 +379,7 @@ gst_d3d11_vp8_dec_start_picture (GstVp8Decoder * decoder,
 
   inner->bitstream_buffer.resize (0);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static ID3D11VideoDecoderOutputView *
@@ -537,7 +537,7 @@ gst_d3d11_vp8_dec_copy_segmentation_params (GstD3D11Vp8Dec * self,
   }
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_vp8_dec_decode_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture, GstVp8Parser * parser)
 {
@@ -553,7 +553,7 @@ gst_d3d11_vp8_dec_decode_picture (GstVp8Decoder * decoder,
       picture, &view_id);
   if (!view) {
     GST_ERROR_OBJECT (self, "current picture does not have output view handle");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   memset (pic_params, 0, sizeof (DXVA_PicParams_VP8));
@@ -575,10 +575,10 @@ gst_d3d11_vp8_dec_decode_picture (GstVp8Decoder * decoder,
   slice->SliceBytesInBuffer = inner->bitstream_buffer.size ();
   slice->wBadSliceChopping = 0;
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_vp8_dec_end_picture (GstVp8Decoder * decoder, GstVp8Picture * picture)
 {
   GstD3D11Vp8Dec *self = GST_D3D11_VP8_DEC (decoder);
@@ -591,14 +591,14 @@ gst_d3d11_vp8_dec_end_picture (GstVp8Decoder * decoder, GstVp8Picture * picture)
 
   if (inner->bitstream_buffer.empty ()) {
     GST_ERROR_OBJECT (self, "No bitstream buffer to submit");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   view = gst_d3d11_vp8_dec_get_output_view_from_picture (self,
       picture, &view_id);
   if (!view) {
     GST_ERROR_OBJECT (self, "current picture does not have output view handle");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   memset (&input_args, 0, sizeof (GstD3D11DecodeInputStreamArgs));
@@ -624,8 +624,10 @@ gst_d3d11_vp8_dec_end_picture (GstVp8Decoder * decoder, GstVp8Picture * picture)
   input_args.bitstream = &inner->bitstream_buffer[0];
   input_args.bitstream_size = inner->bitstream_buffer.size ();
 
-  return gst_d3d11_decoder_decode_frame (inner->d3d11_decoder,
-      view, &input_args);
+  if (!gst_d3d11_decoder_decode_frame (inner->d3d11_decoder, view, &input_args))
+    return GST_FLOW_ERROR;
+
+  return GST_FLOW_OK;
 }
 
 static GstFlowReturn
diff --git a/sys/nvcodec/gstnvvp8dec.c b/sys/nvcodec/gstnvvp8dec.c
index b4cb8f5b0..e2739e09f 100644
--- a/sys/nvcodec/gstnvvp8dec.c
+++ b/sys/nvcodec/gstnvvp8dec.c
@@ -72,11 +72,11 @@ static gboolean gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder,
     GstQuery * query);
 
 /* GstVp8Decoder */
-static gboolean gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
+static GstFlowReturn gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     const GstVp8FrameHdr * frame_hdr);
-static gboolean gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
+static GstFlowReturn gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
     GstVideoCodecFrame * frame, GstVp8Picture * picture);
-static gboolean gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
+static GstFlowReturn gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture, GstVp8Parser * parser);
 static GstFlowReturn gst_nv_vp8_dec_output_picture (GstVp8Decoder *
     decoder, GstVideoCodecFrame * frame, GstVp8Picture * picture);
@@ -225,7 +225,7 @@ gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
   return GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query);
 }
 
-static gboolean
+static GstFlowReturn
 gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     const GstVp8FrameHdr * frame_hdr)
 {
@@ -256,12 +256,12 @@ gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
             cudaVideoCodec_VP8, &info, self->width, self->height,
             NUM_OUTPUT_VIEW)) {
       GST_ERROR_OBJECT (self, "Failed to configure decoder");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
 
     if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
       GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
 
     memset (&self->params, 0, sizeof (CUVIDPICPARAMS));
@@ -273,10 +273,10 @@ gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     self->params.CodecSpecific.vp8.height = self->height;
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
     GstVideoCodecFrame * frame, GstVp8Picture * picture)
 {
@@ -286,7 +286,7 @@ gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
   nv_frame = gst_nv_decoder_new_frame (self->decoder);
   if (!nv_frame) {
     GST_ERROR_OBJECT (self, "No available decoder frame");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   GST_LOG_OBJECT (self,
@@ -295,7 +295,7 @@ gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
   gst_vp8_picture_set_user_data (picture,
       nv_frame, (GDestroyNotify) gst_nv_decoder_frame_unref);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static GstNvDecoderFrame *
@@ -312,7 +312,7 @@ gst_nv_vp8_dec_get_decoder_frame_from_picture (GstNvVp8Dec * self,
   return frame;
 }
 
-static gboolean
+static GstFlowReturn
 gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture, GstVp8Parser * parser)
 {
@@ -327,7 +327,7 @@ gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
   frame = gst_nv_vp8_dec_get_decoder_frame_from_picture (self, picture);
   if (!frame) {
     GST_ERROR_OBJECT (self, "Decoder frame is unavailable");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   self->params.nBitstreamDataLen = picture->size;
@@ -346,7 +346,7 @@ gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
         decoder->alt_ref_picture);
     if (!other_frame) {
       GST_ERROR_OBJECT (self, "Couldn't get decoder frame for AltRef");
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     self->params.CodecSpecific.vp8.AltRefIdx = other_frame->index;
@@ -360,7 +360,7 @@ gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
         decoder->golden_ref_picture);
     if (!other_frame) {
       GST_ERROR_OBJECT (self, "Couldn't get decoder frame for GoldenRef");
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     self->params.CodecSpecific.vp8.GoldenRefIdx = other_frame->index;
@@ -374,7 +374,7 @@ gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
         decoder->last_picture);
     if (!other_frame) {
       GST_ERROR_OBJECT (self, "Couldn't get decoder frame for LastRef");
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     self->params.CodecSpecific.vp8.LastRefIdx = other_frame->index;
@@ -391,7 +391,10 @@ gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
       parser->segmentation.segmentation_enabled ?
       parser->segmentation.update_segment_feature_data : 0;
 
-  return gst_nv_decoder_decode_picture (self->decoder, &self->params);
+  if (!gst_nv_decoder_decode_picture (self->decoder, &self->params))
+    return GST_FLOW_ERROR;
+
+  return GST_FLOW_OK;
 }
 
 static GstFlowReturn
diff --git a/sys/v4l2codecs/gstv4l2codecvp8dec.c b/sys/v4l2codecs/gstv4l2codecvp8dec.c
index 557b31759..fd2a690f5 100644
--- a/sys/v4l2codecs/gstv4l2codecvp8dec.c
+++ b/sys/v4l2codecs/gstv4l2codecvp8dec.c
@@ -435,7 +435,7 @@ gst_v4l2_codec_vp8_dec_fill_references (GstV4l2CodecVp8Dec * self)
       (guint32) self->frame_header.alt_frame_ts / 1000);
 }
 
-static gboolean
+static GstFlowReturn
 gst_v4l2_codec_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     const GstVp8FrameHdr * frame_hdr)
 {
@@ -460,7 +460,7 @@ gst_v4l2_codec_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     self->need_negotiation = TRUE;
     if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
       GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
   }
 
@@ -485,10 +485,10 @@ gst_v4l2_codec_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     self->copy_frames = FALSE;
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_v4l2_codec_vp8_dec_start_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture)
 {
@@ -496,7 +496,7 @@ gst_v4l2_codec_vp8_dec_start_picture (GstVp8Decoder * decoder,
 
   /* FIXME base class should not call us if negotiation failed */
   if (!self->sink_allocator)
-    return FALSE;
+    return GST_FLOW_NOT_NEGOTIATED;
 
   /* Ensure we have a bitstream to write into */
   if (!self->bitstream) {
@@ -505,24 +505,24 @@ gst_v4l2_codec_vp8_dec_start_picture (GstVp8Decoder * decoder,
     if (!self->bitstream) {
       GST_ELEMENT_ERROR (decoder, RESOURCE, NO_SPACE_LEFT,
           ("Not enough memory to decode VP8 stream."), (NULL));
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     if (!gst_memory_map (self->bitstream, &self->bitstream_map, GST_MAP_WRITE)) {
       GST_ELEMENT_ERROR (decoder, RESOURCE, WRITE,
           ("Could not access bitstream memory for writing"), (NULL));
       g_clear_pointer (&self->bitstream, gst_memory_unref);
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
   }
 
   /* We use this field to track how much we have written */
   self->bitstream_map.size = 0;
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_v4l2_codec_vp8_dec_decode_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture, GstVp8Parser * parser)
 {
@@ -532,7 +532,7 @@ gst_v4l2_codec_vp8_dec_decode_picture (GstVp8Decoder * decoder,
   if (self->bitstream_map.maxsize < picture->size) {
     GST_ELEMENT_ERROR (decoder, RESOURCE, NO_SPACE_LEFT,
         ("Not enough space to send picture bitstream."), (NULL));
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   gst_v4l2_codec_vp8_dec_fill_frame_header (self, &picture->frame_hdr);
@@ -545,7 +545,7 @@ gst_v4l2_codec_vp8_dec_decode_picture (GstVp8Decoder * decoder,
   memcpy (bitstream_data, picture->data, picture->size);
   self->bitstream_map.size = picture->size;
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static void
@@ -559,7 +559,7 @@ gst_v4l2_codec_vp8_dec_reset_picture (GstV4l2CodecVp8Dec * self)
   }
 }
 
-static gboolean
+static GstFlowReturn
 gst_v4l2_codec_vp8_dec_end_picture (GstVp8Decoder * decoder,
     GstVp8Picture * picture)
 {
@@ -567,7 +567,7 @@ gst_v4l2_codec_vp8_dec_end_picture (GstVp8Decoder * decoder,
   GstVideoCodecFrame *frame;
   GstV4l2Request *request;
   GstBuffer *buffer;
-  GstFlowReturn flow_ret;
+  GstFlowReturn flow_ret = GST_FLOW_OK;
   gsize bytesused;
 
   /* *INDENT-OFF* */
@@ -598,7 +598,7 @@ gst_v4l2_codec_vp8_dec_end_picture (GstVp8Decoder * decoder,
 
   frame = gst_video_decoder_get_frame (GST_VIDEO_DECODER (self),
       picture->system_frame_number);
-  g_return_val_if_fail (frame, FALSE);
+  g_return_val_if_fail (frame, GST_FLOW_ERROR);
   g_warn_if_fail (frame->output_buffer == NULL);
   frame->output_buffer = buffer;
   gst_video_codec_frame_unref (frame);
@@ -628,11 +628,16 @@ gst_v4l2_codec_vp8_dec_end_picture (GstVp8Decoder * decoder,
   }
 
   gst_v4l2_codec_vp8_dec_reset_picture (self);
-  return TRUE;
+
+  return GST_FLOW_OK;
 
 fail:
   gst_v4l2_codec_vp8_dec_reset_picture (self);
-  return FALSE;
+
+  if (flow_ret != GST_FLOW_OK)
+    return flow_ret;
+
+  return GST_FLOW_ERROR;
 }
 
 static gboolean
diff --git a/sys/va/gstvavp8dec.c b/sys/va/gstvavp8dec.c
index 18fc3cdbf..c8f200174 100644
--- a/sys/va/gstvavp8dec.c
+++ b/sys/va/gstvavp8dec.c
@@ -142,7 +142,7 @@ _get_profile (GstVaVp8Dec * self, const GstVp8FrameHdr * frame_hdr)
   return VAProfileVP8Version0_3;
 }
 
-static gboolean
+static GstFlowReturn
 gst_va_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     const GstVp8FrameHdr * frame_hdr)
 {
@@ -156,12 +156,12 @@ gst_va_vp8_dec_new_sequence (GstVp8Decoder * decoder,
 
   profile = _get_profile (self, frame_hdr);
   if (profile == VAProfileNone)
-    return FALSE;
+    return GST_FLOW_NOT_NEGOTIATED;
 
   if (!gst_va_decoder_has_profile (base->decoder, profile)) {
     GST_ERROR_OBJECT (self, "Profile %s is not supported",
         gst_va_profile_name (profile));
-    return FALSE;
+    return GST_FLOW_NOT_NEGOTIATED;
   }
 
   /* VP8 always use 8 bits 4:2:0 */
@@ -182,14 +182,14 @@ gst_va_vp8_dec_new_sequence (GstVp8Decoder * decoder,
     self->need_negotiation = TRUE;
     if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
       GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_va_vp8_dec_new_picture (GstVp8Decoder * decoder,
     GstVideoCodecFrame * frame, GstVp8Picture * picture)
 {
@@ -209,14 +209,15 @@ gst_va_vp8_dec_new_picture (GstVp8Decoder * decoder,
 
   GST_LOG_OBJECT (self, "New va decode picture %p - %#x", pic,
       gst_va_decode_picture_get_surface (pic));
-  return TRUE;
+
+  return GST_FLOW_OK;
 
 error:
   {
     GST_WARNING_OBJECT (self,
         "Failed to allocated output buffer, return %s",
         gst_flow_get_name (self->last_ret));
-    return FALSE;
+    return self->last_ret;
   }
 }
 
@@ -412,11 +413,14 @@ static gboolean
 gst_va_vp8_dec_decode_picture (GstVp8Decoder * decoder, GstVp8Picture * picture,
     GstVp8Parser * parser)
 {
-  return _fill_picture (decoder, picture, parser) &&
-      _add_slice (decoder, picture, parser);
+  if (_fill_picture (decoder, picture, parser) &&
+      _add_slice (decoder, picture, parser))
+    return GST_FLOW_OK;
+
+  return GST_FLOW_ERROR;
 }
 
-static gboolean
+static GstFlowReturn
 gst_va_vp8_dec_end_picture (GstVp8Decoder * decoder, GstVp8Picture * picture)
 {
   GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
@@ -427,7 +431,10 @@ gst_va_vp8_dec_end_picture (GstVp8Decoder * decoder, GstVp8Picture * picture)
 
   va_pic = gst_vp8_picture_get_user_data (picture);
 
-  return gst_va_decoder_decode (base->decoder, va_pic);
+  if (!gst_va_decoder_decode (base->decoder, va_pic))
+    return GST_FLOW_ERROR;
+
+  return GST_FLOW_OK;
 }
 
 static GstFlowReturn