d3d11videosink: Add support for GstVideoOverlay::set_render_rectangle

Inspired by an MR https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2382

The idea is that we can make use of MoveWindow() in WIN32 d3d11window
implementation safely because WIN32 d3d11window implementation creates
internal HWND even when external HWND is set and then subclassing is used to
draw on internal HWND in any case. So the coordinates passed to MoveWindow()
will be relative to parent HWND, and it meets well to the concept of
set_render_rectangle().

On MoveWindow() event, WM_SIZE event will be generated by OS and then
GstD3D11WindowWin32 implementation will update render area including swapchain
correspondingly, as if it's normal window move/resize case.

But in case of UWP (CoreWindow or SwapChainPanel), we need more research to
meet expected behavior of set_render_rectangle()

Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/issues/1416
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2450>

4 files changed, 95 insertions, 18 deletions

diff --git a/sys/d3d11/gstd3d11videosink.cpp b/sys/d3d11/gstd3d11videosink.cpp
index 07da3ee98..dee6b0fad 100644
--- a/sys/d3d11/gstd3d11videosink.cpp
+++ b/sys/d3d11/gstd3d11videosink.cpp
@@ -579,18 +579,16 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
     goto no_display_size;
 
   GST_OBJECT_LOCK (self);
-  if (!self->pending_render_rect) {
-    self->render_rect.x = 0;
-    self->render_rect.y = 0;
-    self->render_rect.w = GST_VIDEO_SINK_WIDTH (self);
-    self->render_rect.h = GST_VIDEO_SINK_HEIGHT (self);
-  }
+  if (self->pending_render_rect) {
+    GstVideoRectangle rect = self->render_rect;
 
-  gst_d3d11_window_set_render_rectangle (self->window,
-      self->render_rect.x, self->render_rect.y, self->render_rect.w,
-      self->render_rect.h);
-  self->pending_render_rect = FALSE;
-  GST_OBJECT_UNLOCK (self);
+    self->pending_render_rect = FALSE;
+    GST_OBJECT_UNLOCK (self);
+
+    gst_d3d11_window_set_render_rectangle (self->window, &rect);
+  } else {
+    GST_OBJECT_UNLOCK (self);
+  }
 
   self->have_video_processor = FALSE;
   if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self),
@@ -1223,15 +1221,25 @@ gst_d3d11_video_sink_set_render_rectangle (GstVideoOverlay * overlay, gint x,
 
   GST_OBJECT_LOCK (self);
   if (self->window) {
-    gst_d3d11_window_set_render_rectangle (self->window, x, y, width, height);
+    GstVideoRectangle rect;
+
+    rect.x = x;
+    rect.y = y;
+    rect.w = width;
+    rect.h = height;
+
+    self->render_rect = rect;
+    GST_OBJECT_UNLOCK (self);
+
+    gst_d3d11_window_set_render_rectangle (self->window, &rect);
   } else {
     self->render_rect.x = x;
     self->render_rect.y = y;
     self->render_rect.w = width;
     self->render_rect.h = height;
     self->pending_render_rect = TRUE;
+    GST_OBJECT_UNLOCK (self);
   }
-  GST_OBJECT_UNLOCK (self);
 }
 
 static void
diff --git a/sys/d3d11/gstd3d11window.cpp b/sys/d3d11/gstd3d11window.cpp
index 4be0fb887..6087b8df4 100644
--- a/sys/d3d11/gstd3d11window.cpp
+++ b/sys/d3d11/gstd3d11window.cpp
@@ -784,12 +784,17 @@ gst_d3d11_window_show (GstD3D11Window * window)
 }
 
 void
-gst_d3d11_window_set_render_rectangle (GstD3D11Window * window, gint x, gint y,
-    gint width, gint height)
+gst_d3d11_window_set_render_rectangle (GstD3D11Window * window,
+    const GstVideoRectangle * rect)
 {
+  GstD3D11WindowClass *klass;
+
   g_return_if_fail (GST_IS_D3D11_WINDOW (window));
 
-  /* TODO: resize window and view */
+  klass = GST_D3D11_WINDOW_GET_CLASS (window);
+
+  if (klass->set_render_rectangle)
+    klass->set_render_rectangle (window, rect);
 }
 
 static gboolean
diff --git a/sys/d3d11/gstd3d11window.h b/sys/d3d11/gstd3d11window.h
index d0371d9bd..a03337edb 100644
--- a/sys/d3d11/gstd3d11window.h
+++ b/sys/d3d11/gstd3d11window.h
@@ -167,6 +167,9 @@ struct _GstD3D11WindowClass
 
   gboolean      (*release_shared_handle)  (GstD3D11Window * window,
                                            GstD3D11WindowSharedHandleData * data);
+
+  void          (*set_render_rectangle)   (GstD3D11Window * window,
+                                           const GstVideoRectangle * rect);
 };
 
 GType         gst_d3d11_window_get_type             (void);
@@ -174,8 +177,7 @@ GType         gst_d3d11_window_get_type             (void);
 void          gst_d3d11_window_show                 (GstD3D11Window * window);
 
 void          gst_d3d11_window_set_render_rectangle (GstD3D11Window * window,
-                                                     gint x, gint y,
-                                                     gint width, gint height);
+                                                     const GstVideoRectangle * rect);
 
 gboolean      gst_d3d11_window_prepare              (GstD3D11Window * window,
                                                      guint display_width,
diff --git a/sys/d3d11/gstd3d11window_win32.cpp b/sys/d3d11/gstd3d11window_win32.cpp
index 01dfcbc48..039263f0a 100644
--- a/sys/d3d11/gstd3d11window_win32.cpp
+++ b/sys/d3d11/gstd3d11window_win32.cpp
@@ -43,6 +43,7 @@ G_LOCK_DEFINE_STATIC (create_lock);
 #define WM_GST_D3D11_FULLSCREEN (WM_USER + 1)
 #define WM_GST_D3D11_CONSTRUCT_INTERNAL_WINDOW (WM_USER + 2)
 #define WM_GST_D3D11_DESTROY_INTERNAL_WINDOW (WM_USER + 3)
+#define WM_GST_D3D11_MOVE_WINDOW (WM_USER + 4)
 
 static LRESULT CALLBACK window_proc (HWND hWnd, UINT uMsg, WPARAM wParam,
     LPARAM lParam);
@@ -85,10 +86,14 @@ struct _GstD3D11WindowWin32
 
   /* atomic */
   gint pending_fullscreen_count;
+  gint pending_move_window;
 
   /* fullscreen related */
   RECT restore_rect;
   LONG restore_style;
+
+  /* Handle set_render_rectangle */
+  GstVideoRectangle render_rect;
 };
 
 #define gst_d3d11_window_win32_parent_class parent_class
@@ -122,6 +127,9 @@ static void
 gst_d3d11_window_win32_on_resize (GstD3D11Window * window,
     guint width, guint height);
 static void gst_d3d11_window_win32_unprepare (GstD3D11Window * window);
+static void
+gst_d3d11_window_win32_set_render_rectangle (GstD3D11Window * window,
+    const GstVideoRectangle * rect);
 
 static void
 gst_d3d11_window_win32_class_init (GstD3D11WindowWin32Class * klass)
@@ -145,6 +153,8 @@ gst_d3d11_window_win32_class_init (GstD3D11WindowWin32Class * klass)
       GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_on_resize);
   window_class->unprepare =
       GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_unprepare);
+  window_class->set_render_rectangle =
+      GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_set_render_rectangle);
 }
 
 static void
@@ -241,6 +251,37 @@ gst_d3d11_window_win32_unprepare (GstD3D11Window * window)
 }
 
 static void
+gst_d3d11_window_win32_set_render_rectangle (GstD3D11Window * window,
+    const GstVideoRectangle * rect)
+{
+  GstD3D11WindowWin32 *self = GST_D3D11_WINDOW_WIN32 (window);
+
+  if (self->external_hwnd && self->internal_hwnd) {
+    g_atomic_int_add (&self->pending_move_window, 1);
+    self->render_rect = *rect;
+
+    if (self->internal_hwnd_thread == g_thread_self ()) {
+      /* We are on message pumping thread already, handle this synchroniously */
+      SendMessage (self->internal_hwnd, WM_GST_D3D11_MOVE_WINDOW, 0, 0);
+    } else {
+      /* Post message to message pumping thread. Handling HWND specific message
+       * on message pumping thread is not a worst idea in generall */
+      PostMessage (self->internal_hwnd, WM_GST_D3D11_MOVE_WINDOW, 0, 0);
+    }
+  } else {
+    /* XXX: Not sure what's expected behavior if we are drawing on internal
+     * HWND but user wants to specify rectangle.
+     *
+     * - Should we move window to corresponding desktop coordinates ?
+     * - Or should crop correspondingly by modifying viewport of
+     *   render target view of swapchian's backbuffer or so ?
+     * - Or should we ignore set_render_rectangle if we are drawing on
+     *   internal HWND without external HWND ?
+     */
+  }
+}
+
+static void
 gst_d3d11_window_win32_finalize (GObject * object)
 {
   GstD3D11WindowWin32 *self = GST_D3D11_WINDOW_WIN32 (object);
@@ -663,6 +704,27 @@ gst_d3d11_window_win32_handle_window_proc (GstD3D11WindowWin32 * self,
           gst_d3d11_window_win32_change_fullscreen_mode_internal (self);
       }
       break;
+    case WM_GST_D3D11_MOVE_WINDOW:
+      if (g_atomic_int_get (&self->pending_move_window)) {
+        g_atomic_int_set (&self->pending_move_window, 0);
+
+        if (self->internal_hwnd && self->external_hwnd) {
+          if (self->render_rect.w < 0 || self->render_rect.h < 0) {
+            RECT rect;
+
+            /* Reset render rect and back to full-size window */
+            if (GetClientRect (self->external_hwnd, &rect)) {
+              MoveWindow (self->internal_hwnd, 0, 0,
+                  rect.right - rect.left, rect.bottom - rect.top, FALSE);
+            }
+          } else {
+            MoveWindow (self->internal_hwnd, self->render_rect.x,
+                self->render_rect.y, self->render_rect.w, self->render_rect.h,
+                FALSE);
+          }
+        }
+      }
+      break;
     default:
       break;
   }