avfilter/all: propagate errors of functions from avfilter/formats

Many of the functions from avfilter/formats can return errors, usually AVERROR(ENOMEM).
This propagates the return values.

All of these were found by using av_warn_unused_result, demonstrating its utility.

Tested with FATE. I am least sure of the changes to avfilter/filtergraph,
since I don't know what/how reduce_format is intended to behave and how it should
react to errors.

Fixes: CID 1325680, 1325679, 1325678.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Previous version Reviewed-by: Nicolas George <george@nsup.org>
Previous version Reviewed-by: Clément Bœsch <u@pkh.me>
Signed-off-by: Ganesh Ajjanagadde <gajjanagadde@gmail.com>

1 files changed, 5 insertions, 3 deletions

diff --git a/libavfilter/vf_vectorscope.c b/libavfilter/vf_vectorscope.c
index 730f55f5fb..5f7cd56565 100644
--- a/libavfilter/vf_vectorscope.c
+++ b/libavfilter/vf_vectorscope.c
@@ -140,7 +140,7 @@ static int query_formats(AVFilterContext *ctx)
     const enum AVPixelFormat *out_pix_fmts;
     const AVPixFmtDescriptor *desc;
     AVFilterFormats *avff;
-    int depth, rgb, i;
+    int depth, rgb, i, ret;
 
     if (!ctx->inputs[0]->in_formats ||
         !ctx->inputs[0]->in_formats->nb_formats) {
@@ -154,7 +154,8 @@ static int query_formats(AVFilterContext *ctx)
             in_pix_fmts = in2_pix_fmts;
         else
             in_pix_fmts = in1_pix_fmts;
-        ff_formats_ref(ff_make_format_list(in_pix_fmts), &ctx->inputs[0]->out_formats);
+        if ((ret = ff_formats_ref(ff_make_format_list(in_pix_fmts), &ctx->inputs[0]->out_formats)) < 0)
+            return ret;
     }
 
     avff = ctx->inputs[0]->in_formats;
@@ -180,7 +181,8 @@ static int query_formats(AVFilterContext *ctx)
         out_pix_fmts = out_yuv10_pix_fmts;
     else
         out_pix_fmts = out_yuv8_pix_fmts;
-    ff_formats_ref(ff_make_format_list(out_pix_fmts), &ctx->outputs[0]->in_formats);
+    if ((ret = ff_formats_ref(ff_make_format_list(out_pix_fmts), &ctx->outputs[0]->in_formats)) < 0)
+        return ret;
 
     return 0;
 }