Client: add custom margins support to xdg popups

xdg_positioner operates in window content geometry (i.e. without shadows),
so it's necessary to remove popup's and parent's shadow from position and
size calculations.

Change-Id: I997ba2c45ae341a09a8284226629e8107800e894
Reviewed-by: David Edmundson <davidedmundson@kde.org>
(cherry picked from commit 2a4dc215756ff2f3fe966a689eea4cf4c7b11b7d)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

3 files changed, 11 insertions, 3 deletions

diff --git a/src/client/qwaylandwindow.cpp b/src/client/qwaylandwindow.cpp
index 53ebbe75..9c6ce91a 100644
--- a/src/client/qwaylandwindow.cpp
+++ b/src/client/qwaylandwindow.cpp
@@ -728,6 +728,11 @@ QMargins QWaylandWindow::clientSideMargins() const
     return mWindowDecorationEnabled ? mWindowDecoration->margins() : QMargins{};
 }
 
+QMargins QWaylandWindow::customMargins() const
+{
+    return mCustomMargins;
+}
+
 void QWaylandWindow::setCustomMargins(const QMargins &margins) {
     const QMargins oldMargins = mCustomMargins;
     mCustomMargins = margins;
diff --git a/src/client/qwaylandwindow_p.h b/src/client/qwaylandwindow_p.h
index ea30d9b8..298f3f2d 100644
--- a/src/client/qwaylandwindow_p.h
+++ b/src/client/qwaylandwindow_p.h
@@ -112,6 +112,7 @@ public:
     bool waitForFrameSync(int timeout);
 
     QMargins frameMargins() const override;
+    QMargins customMargins() const;
     void setCustomMargins(const QMargins &margins);
     QSize surfaceSize() const;
     QRect windowContentGeometry() const;
diff --git a/src/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp b/src/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp
index c4b1942b..415bcad2 100644
--- a/src/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp
+++ b/src/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp
@@ -407,8 +407,10 @@ void QWaylandXdgSurface::setPopup(QWaylandWindow *parent)
 
     auto positioner = new QtWayland::xdg_positioner(m_shell->create_positioner());
     // set_popup expects a position relative to the parent
-    QPoint transientPos = m_window->geometry().topLeft(); // this is absolute
-    transientPos -= parent->geometry().topLeft();
+    QPoint topLeftMargins = QPoint(m_window->customMargins().left(), m_window->customMargins().top());
+    QPoint parentMargins = QPoint(parent->customMargins().left(), parent->customMargins().top());
+    QPoint transientPos = m_window->geometry().topLeft() + topLeftMargins; // this is absolute
+    transientPos -= parent->geometry().topLeft() + parentMargins;
     if (parent->decoration()) {
         transientPos.setX(transientPos.x() + parent->decoration()->margins(QWaylandAbstractDecoration::ShadowsExcluded).left());
         transientPos.setY(transientPos.y() + parent->decoration()->margins(QWaylandAbstractDecoration::ShadowsExcluded).top());
@@ -416,7 +418,7 @@ void QWaylandXdgSurface::setPopup(QWaylandWindow *parent)
     positioner->set_anchor_rect(transientPos.x(), transientPos.y(), 1, 1);
     positioner->set_anchor(QtWayland::xdg_positioner::anchor_top_left);
     positioner->set_gravity(QtWayland::xdg_positioner::gravity_bottom_right);
-    positioner->set_size(m_window->geometry().width(), m_window->geometry().height());
+    positioner->set_size(m_window->windowContentGeometry().width(), m_window->windowContentGeometry().height());
     positioner->set_constraint_adjustment(QtWayland::xdg_positioner::constraint_adjustment_slide_x
         | QtWayland::xdg_positioner::constraint_adjustment_slide_y
         | QtWayland::xdg_positioner::constraint_adjustment_flip_x