lavfi/extractplanes: Fix in_pixfmts.

If the original pix_fmt was >8 bit and not supported by the filter,
the filter system could choose a pix_fmt with different endianness
as input for extractplanes which broke the output because the output
always used the endianness of the original pix_fmt.

1 files changed, 32 insertions, 10 deletions

diff --git a/libavfilter/vf_extractplanes.c b/libavfilter/vf_extractplanes.c
index dc37ca8ff7..320305781a 100644
--- a/libavfilter/vf_extractplanes.c
+++ b/libavfilter/vf_extractplanes.c
@@ -62,7 +62,7 @@ AVFILTER_DEFINE_CLASS(extractplanes);
 
 static int query_formats(AVFilterContext *ctx)
 {
-    static const enum AVPixelFormat in_pixfmts[] = {
+    static const enum AVPixelFormat in_pixfmts_le[] = {
         AV_PIX_FMT_YUV410P,
         AV_PIX_FMT_YUV411P,
         AV_PIX_FMT_YUV440P,
@@ -73,30 +73,47 @@ static int query_formats(AVFilterContext *ctx)
         AV_PIX_FMT_YUVJ411P,
         AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVA444P,
         AV_PIX_FMT_YUV420P16LE, AV_PIX_FMT_YUVA420P16LE,
-        AV_PIX_FMT_YUV420P16BE, AV_PIX_FMT_YUVA420P16BE,
         AV_PIX_FMT_YUV422P16LE, AV_PIX_FMT_YUVA422P16LE,
-        AV_PIX_FMT_YUV422P16BE, AV_PIX_FMT_YUVA422P16BE,
         AV_PIX_FMT_YUV444P16LE, AV_PIX_FMT_YUVA444P16LE,
-        AV_PIX_FMT_YUV444P16BE, AV_PIX_FMT_YUVA444P16BE,
         AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY8A,
-        AV_PIX_FMT_YA16LE, AV_PIX_FMT_YA16BE,
-        AV_PIX_FMT_GRAY16LE, AV_PIX_FMT_GRAY16BE,
+        AV_PIX_FMT_YA16LE, AV_PIX_FMT_GRAY16LE,
         AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
         AV_PIX_FMT_RGBA, AV_PIX_FMT_BGRA,
         AV_PIX_FMT_ARGB, AV_PIX_FMT_ABGR,
         AV_PIX_FMT_RGB48LE, AV_PIX_FMT_BGR48LE,
-        AV_PIX_FMT_RGB48BE, AV_PIX_FMT_BGR48BE,
         AV_PIX_FMT_RGBA64LE, AV_PIX_FMT_BGRA64LE,
+        AV_PIX_FMT_GBRP, AV_PIX_FMT_GBRAP,
+        AV_PIX_FMT_GBRP16LE, AV_PIX_FMT_GBRAP16LE,
+        AV_PIX_FMT_NONE,
+    };
+    static const enum AVPixelFormat in_pixfmts_be[] = {
+        AV_PIX_FMT_YUV410P,
+        AV_PIX_FMT_YUV411P,
+        AV_PIX_FMT_YUV440P,
+        AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVA420P,
+        AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUVA422P,
+        AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P,
+        AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P,
+        AV_PIX_FMT_YUVJ411P,
+        AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVA444P,
+        AV_PIX_FMT_YUV420P16BE, AV_PIX_FMT_YUVA420P16BE,
+        AV_PIX_FMT_YUV422P16BE, AV_PIX_FMT_YUVA422P16BE,
+        AV_PIX_FMT_YUV444P16BE, AV_PIX_FMT_YUVA444P16BE,
+        AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY8A,
+        AV_PIX_FMT_YA16BE, AV_PIX_FMT_GRAY16BE,
+        AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
+        AV_PIX_FMT_RGBA, AV_PIX_FMT_BGRA,
+        AV_PIX_FMT_ARGB, AV_PIX_FMT_ABGR,
+        AV_PIX_FMT_RGB48BE, AV_PIX_FMT_BGR48BE,
         AV_PIX_FMT_RGBA64BE, AV_PIX_FMT_BGRA64BE,
         AV_PIX_FMT_GBRP, AV_PIX_FMT_GBRAP,
-        AV_PIX_FMT_GBRP16LE, AV_PIX_FMT_GBRP16BE,
-        AV_PIX_FMT_GBRAP16LE, AV_PIX_FMT_GBRAP16BE,
+        AV_PIX_FMT_GBRP16BE, AV_PIX_FMT_GBRAP16BE,
         AV_PIX_FMT_NONE,
     };
     static const enum AVPixelFormat out8_pixfmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE };
     static const enum AVPixelFormat out16le_pixfmts[] = { AV_PIX_FMT_GRAY16LE, AV_PIX_FMT_NONE };
     static const enum AVPixelFormat out16be_pixfmts[] = { AV_PIX_FMT_GRAY16BE, AV_PIX_FMT_NONE };
-    const enum AVPixelFormat *out_pixfmts;
+    const enum AVPixelFormat *out_pixfmts, *in_pixfmts;
     const AVPixFmtDescriptor *desc;
     AVFilterFormats *avff;
     int i, ret, depth = 0, be = 0;
@@ -110,6 +127,11 @@ static int query_formats(AVFilterContext *ctx)
     desc = av_pix_fmt_desc_get(avff->formats[0]);
     depth = desc->comp[0].depth;
     be = desc->flags & AV_PIX_FMT_FLAG_BE;
+    if (be) {
+        in_pixfmts = in_pixfmts_be;
+    } else {
+        in_pixfmts = in_pixfmts_le;
+    }
     if (!ctx->inputs[0]->out_formats)
         if ((ret = ff_formats_ref(ff_make_format_list(in_pixfmts), &ctx->inputs[0]->out_formats)) < 0)
             return ret;