From 6db6a8c7d9ffcf2b7e226b7e874ee67b82f24ac8 Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha@centricular.com>
Date: Wed, 28 Oct 2020 00:47:49 +0900
Subject: d3d11videoprocessor: Fix wrong input/output supportability check

The flag argument of ID3D11VideoProcessorEnumerator::CheckVideoProcessorFormat
method is output value, not input.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1736>
---
 sys/d3d11/gstd3d11videoprocessor.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/sys/d3d11/gstd3d11videoprocessor.c b/sys/d3d11/gstd3d11videoprocessor.c
index ed2bd748a..94e601dfe 100644
--- a/sys/d3d11/gstd3d11videoprocessor.c
+++ b/sys/d3d11/gstd3d11videoprocessor.c
@@ -190,20 +190,25 @@ gst_d3d11_video_processor_supports_format (GstD3D11VideoProcessor *
     self, DXGI_FORMAT format, gboolean is_input)
 {
   HRESULT hr;
-  UINT flag;
+  UINT flag = 0;
+
+  hr = ID3D11VideoProcessorEnumerator_CheckVideoProcessorFormat
+      (self->enumerator, format, &flag);
+
+  if (!gst_d3d11_result (hr, self->device))
+    return FALSE;
 
   if (is_input) {
     /* D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT, missing in mingw header */
-    flag = 1;
+    if ((flag & 0x1) != 0)
+      return TRUE;
   } else {
     /* D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT, missing in mingw header */
-    flag = 2;
+    if ((flag & 0x2) != 0)
+      return TRUE;
   }
 
-  hr = ID3D11VideoProcessorEnumerator_CheckVideoProcessorFormat
-      (self->enumerator, format, &flag);
-
-  return gst_d3d11_result (hr, self->device);
+  return FALSE;
 }
 
 gboolean
-- 
cgit v1.2.1