d3d11desktopdup: Don't ignore error DXGI_ERROR_UNSUPPORTED

Although Microsoft's DXGIDesktopDuplication example is considering
the DXGI_ERROR_UNSUPPORTED as an expected error
(See https://github.com/microsoft/Windows-classic-samples/tree/master/Samples/DXGIDesktopDuplication)
it might not be recoverable error if application is
run against a discrete GPU
(See https://docs.microsoft.com/en-US/troubleshoot/windows-client/shell-experience/error-when-dda-capable-app-is-against-gpu)

Do early error out if the error happens while opening device,
instead of retrying it forever.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2208>

3 files changed, 56 insertions, 7 deletions

diff --git a/sys/d3d11/gstd3d11desktopdup.cpp b/sys/d3d11/gstd3d11desktopdup.cpp
index 43bcb0871..2d8bce767 100644
--- a/sys/d3d11/gstd3d11desktopdup.cpp
+++ b/sys/d3d11/gstd3d11desktopdup.cpp
@@ -140,7 +140,6 @@ HRESULT SystemTransitionsExpectedErrors[] = {
 HRESULT CreateDuplicationExpectedErrors[] = {
   DXGI_ERROR_DEVICE_REMOVED,
   static_cast<HRESULT>(E_ACCESSDENIED),
-  DXGI_ERROR_UNSUPPORTED,
   DXGI_ERROR_SESSION_DISCONNECTED,
   S_OK
 };
@@ -680,6 +679,17 @@ private:
         return GST_FLOW_ERROR;
       }
 
+      /* Seems to be one limitation of Desktop Duplication API design
+       * See
+       * https://docs.microsoft.com/en-US/troubleshoot/windows-client/shell-experience/error-when-dda-capable-app-is-against-gpu
+       */
+      if (hr == DXGI_ERROR_UNSUPPORTED) {
+        GST_WARNING ("IDXGIOutput1::DuplicateOutput returned "
+            "DXGI_ERROR_UNSUPPORTED, possiblely application is run against a "
+            "discrete GPU");
+        return GST_D3D11_DESKTOP_DUP_FLOW_UNSUPPORTED;
+      }
+
       return gst_d3d11_desktop_dup_return_from_hr (d3d11_device.Get(), hr,
           CreateDuplicationExpectedErrors);
     }
diff --git a/sys/d3d11/gstd3d11desktopdup.h b/sys/d3d11/gstd3d11desktopdup.h
index 83621c4e1..b56ee9187 100644
--- a/sys/d3d11/gstd3d11desktopdup.h
+++ b/sys/d3d11/gstd3d11desktopdup.h
@@ -28,6 +28,7 @@
 G_BEGIN_DECLS
 
 #define GST_D3D11_DESKTOP_DUP_FLOW_EXPECTED_ERROR GST_FLOW_CUSTOM_SUCCESS
+#define GST_D3D11_DESKTOP_DUP_FLOW_UNSUPPORTED GST_FLOW_CUSTOM_ERROR
 
 #define GST_TYPE_D3D11_DESKTOP_DUP (gst_d3d11_desktop_dup_get_type())
 G_DECLARE_FINAL_TYPE (GstD3D11DesktopDup, gst_d3d11_desktop_dup,
diff --git a/sys/d3d11/gstd3d11desktopdupsrc.cpp b/sys/d3d11/gstd3d11desktopdupsrc.cpp
index 372addc82..eb098cf34 100644
--- a/sys/d3d11/gstd3d11desktopdupsrc.cpp
+++ b/sys/d3d11/gstd3d11desktopdupsrc.cpp
@@ -401,6 +401,7 @@ static gboolean
 gst_d3d11_desktop_dup_src_start (GstBaseSrc * bsrc)
 {
   GstD3D11DesktopDupSrc *self = GST_D3D11_DESKTOP_DUP_SRC (bsrc);
+  GstFlowReturn ret;
 
   /* FIXME: this element will use only the first adapter, but
    * this might cause issue in case of multi-gpu environment and
@@ -415,18 +416,42 @@ gst_d3d11_desktop_dup_src_start (GstBaseSrc * bsrc)
   }
 
   self->dupl = gst_d3d11_desktop_dup_new (self->device, self->monitor_index);
-  if (!self->dupl) {
-    GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND,
-        ("Failed to prepare duplication for output index %d",
-            self->monitor_index), (NULL));
-
-    return FALSE;
+  if (!self->dupl)
+    goto error;
+
+  /* Check if we can open device */
+  ret = gst_d3d11_desktop_dup_prepare (self->dupl);
+  switch (ret) {
+    case GST_D3D11_DESKTOP_DUP_FLOW_EXPECTED_ERROR:
+    case GST_FLOW_OK:
+      break;
+    case GST_D3D11_DESKTOP_DUP_FLOW_UNSUPPORTED:
+      goto unsupported;
+    default:
+      goto error;
   }
 
   self->last_frame_no = -1;
   self->min_latency = self->max_latency = GST_CLOCK_TIME_NONE;
 
   return TRUE;
+
+error:
+  {
+    GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND,
+        ("Failed to prepare duplication for output index %d",
+            self->monitor_index), (NULL));
+  }
+  return FALSE;
+
+unsupported:
+  {
+    GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ,
+        ("Failed to prepare duplication for output index %d",
+            self->monitor_index),
+        ("Try run the application on the integrated GPU"));
+    return FALSE;
+  }
 }
 
 static gboolean
@@ -515,6 +540,8 @@ gst_d3d11_desktop_dup_src_fill (GstPushSrc * pushsrc, GstBuffer * buffer)
   gboolean update_latency = FALSE;
   guint64 next_frame_no;
   gboolean draw_mouse;
+  /* Just magic number... */
+  gint unsupported_retry_count = 100;
 
   if (!self->dupl) {
     GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ,
@@ -635,6 +662,17 @@ again:
     GST_WARNING_OBJECT (self, "Got expected error, try again");
     gst_clear_object (&clock);
     goto again;
+  } else if (ret == GST_D3D11_DESKTOP_DUP_FLOW_UNSUPPORTED) {
+    GST_WARNING_OBJECT (self, "Got DXGI_ERROR_UNSUPPORTED error");
+    unsupported_retry_count--;
+
+    if (unsupported_retry_count < 0) {
+      ret = GST_FLOW_ERROR;
+      goto out;
+    }
+
+    gst_clear_object (&clock);
+    goto again;
   }
 
   after_capture = gst_clock_get_time (clock);