Move the wayland socket polling to a separate event thread

New event threads is introduced which calls poll() on the wayland fd,
instead of relying on the event dispatcher by using the QSocketNotifier.
This allows to call in the proper order the wl_display_prepare_read(),
poll() and wl_display_read_events() functions.

One thread is responsible for the default queue; when needed, it emit
a signal so that the main thread can dispatch the queue. Another thread
is responsible for the dedicated queue for frame callbacks; this thread
will dispatch events on the thread itself.

QWaylandWindow is updated to, instead of each window's dedicated event
queue, use this queue for frame callbacks.

Co-authored-by: Ratchanan Srirattanamet <ratchanan@ubports.com>
Task-number: QTBUG-66075
Change-Id: Ibb33ad7f4193b866d1b8d7a0405a94d59dcad5eb
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Eskil Abrahamsen Blomfeldt <eskil.abrahamsen-blomfeldt@qt.io>

1 files changed, 217 insertions, 84 deletions

diff --git a/src/client/qwaylanddisplay.cpp b/src/client/qwaylanddisplay.cpp
index 9ae6abd9..c56351ca 100644
--- a/src/client/qwaylanddisplay.cpp
+++ b/src/client/qwaylanddisplay.cpp
@@ -90,10 +90,201 @@
 
 #include <tuple> // for std::tie
 
+static void checkWaylandError(struct wl_display *display)
+{
+    int ecode = wl_display_get_error(display);
+    if ((ecode == EPIPE || ecode == ECONNRESET)) {
+        // special case this to provide a nicer error
+        qWarning("The Wayland connection broke. Did the Wayland compositor die?");
+    } else {
+        qWarning("The Wayland connection experienced a fatal error: %s", strerror(ecode));
+    }
+    _exit(1);
+}
+
 QT_BEGIN_NAMESPACE
 
 namespace QtWaylandClient {
 
+class EventThread : public QThread
+{
+    Q_OBJECT
+public:
+    enum OperatingMode {
+        EmitToDispatch, // Emit the signal, allow dispatching in a differnt thread.
+        SelfDispatch, // Dispatch the events inside this thread.
+    };
+
+    EventThread(struct wl_display * wl, struct wl_event_queue * ev_queue,
+                OperatingMode mode)
+        : m_fd(wl_display_get_fd(wl))
+        , m_pipefd{ -1, -1 }
+        , m_wldisplay(wl)
+        , m_wlevqueue(ev_queue)
+        , m_mode(mode)
+        , m_reading(true)
+        , m_quitting(false)
+    {
+        setObjectName(QStringLiteral("WaylandEventThread"));
+    }
+
+    void readAndDispatchEvents()
+    {
+        /*
+         * Dispatch pending events and flush the requests at least once. If the event thread
+         * is not reading, try to call _prepare_read() to allow the event thread to poll().
+         * If that fails, re-try dispatch & flush again until _prepare_read() is successful.
+         *
+         * This allow any call to readAndDispatchEvents() to start event thread's polling,
+         * not only the one issued from event thread's waitForReading(), which means functions
+         * called from dispatch_pending() can safely spin an event loop.
+         */
+        for (;;) {
+            if (dispatchQueuePending() < 0) {
+                checkWaylandError(m_wldisplay);
+                return;
+            }
+
+            wl_display_flush(m_wldisplay);
+
+            // We have to check if event thread is reading every time we dispatch
+            // something, as that may recursively call this function.
+            if (m_reading.loadAcquire())
+                break;
+
+            if (prepareReadQueue() == 0) {
+                QMutexLocker l(&m_mutex);
+                m_reading.storeRelease(true);
+                m_cond.wakeOne();
+                break;
+            }
+        }
+    }
+
+    void stop()
+    {
+        // We have to both write to the pipe and set the flag, as the thread may be
+        // either in the poll() or waiting for _prepare_read().
+        if (m_pipefd[1] != -1 && write(m_pipefd[1], "\0", 1) == -1)
+            qWarning("Failed to write to the pipe: %s.", strerror(errno));
+
+        {
+            QMutexLocker l(&m_mutex);
+            m_quitting = true;
+            m_cond.wakeOne();
+        }
+
+        wait();
+    }
+
+Q_SIGNALS:
+    void needReadAndDispatch();
+
+protected:
+    void run() override
+    {
+        // we use this pipe to make the loop exit otherwise if we simply used a flag on the loop condition, if stop() gets
+        // called while poll() is blocking the thread will never quit since there are no wayland messages coming anymore.
+        struct Pipe
+        {
+            Pipe(int *fds)
+                : fds(fds)
+            {
+                if (qt_safe_pipe(fds) != 0)
+                    qWarning("Pipe creation failed. Quitting may hang.");
+            }
+            ~Pipe()
+            {
+                if (fds[0] != -1) {
+                    close(fds[0]);
+                    close(fds[1]);
+                }
+            }
+
+            int *fds;
+        } pipe(m_pipefd);
+
+        // Make the main thread call wl_prepare_read(), dispatch the pending messages and flush the
+        // outbound ones. Wait until it's done before proceeding, unless we're told to quit.
+        while (waitForReading()) {
+            pollfd fds[2] = { { m_fd, POLLIN, 0 }, { m_pipefd[0], POLLIN, 0 } };
+            poll(fds, 2, -1);
+
+            if (fds[1].revents & POLLIN) {
+                // we don't really care to read the byte that was written here since we're closing down
+                wl_display_cancel_read(m_wldisplay);
+                break;
+            }
+
+            if (fds[0].revents & POLLIN)
+                wl_display_read_events(m_wldisplay);
+                // The polll was succesfull and the event thread did the wl_display_read_events(). On the next iteration of the loop
+                // the event sent to the main thread will cause it to dispatch the messages just read, unless the loop exits in which
+                // case we don't care anymore about them.
+            else
+                wl_display_cancel_read(m_wldisplay);
+        }
+    }
+
+private:
+    bool waitForReading()
+    {
+        Q_ASSERT(QThread::currentThread() == this);
+
+        m_reading.storeRelease(false);
+
+        if (m_mode == SelfDispatch) {
+            readAndDispatchEvents();
+        } else {
+            Q_EMIT needReadAndDispatch();
+
+            QMutexLocker lock(&m_mutex);
+            // m_reading might be set from our emit or some other invocation of
+            // readAndDispatchEvents().
+            while (!m_reading.loadRelaxed() && !m_quitting)
+                m_cond.wait(&m_mutex);
+        }
+
+        return !m_quitting;
+    }
+
+    int dispatchQueuePending()
+    {
+        if (m_wlevqueue)
+            return wl_display_dispatch_queue_pending(m_wldisplay, m_wlevqueue);
+        else
+            return wl_display_dispatch_pending(m_wldisplay);
+    }
+
+    int prepareReadQueue()
+    {
+        if (m_wlevqueue)
+            return wl_display_prepare_read_queue(m_wldisplay, m_wlevqueue);
+        else
+            return wl_display_prepare_read(m_wldisplay);
+    }
+
+    int m_fd;
+    int m_pipefd[2];
+    wl_display *m_wldisplay;
+    wl_event_queue *m_wlevqueue;
+    OperatingMode m_mode;
+
+    /* Concurrency note when operating in EmitToDispatch mode:
+     * m_reading is set to false inside event thread's waitForReading(), and is
+     * set to true inside main thread's readAndDispatchEvents().
+     * The lock is not taken when setting m_reading to false, as the main thread
+     * is not actively waiting for it to turn false. However, the lock is taken
+     * inside readAndDispatchEvents() before setting m_reading to true,
+     * as the event thread is actively waiting for it under the wait condition.
+     */
+
+    QAtomicInteger<bool> m_reading;
+    bool m_quitting;
+    QMutex m_mutex;
+    QWaitCondition m_cond;
+};
+
 Q_LOGGING_CATEGORY(lcQpaWayland, "qt.qpa.wayland"); // for general (uncategorized) Wayland platform logging
 
 struct wl_surface *QWaylandDisplay::createSurface(void *handle)
@@ -167,6 +358,12 @@ QWaylandDisplay::QWaylandDisplay(QWaylandIntegration *waylandIntegration)
 
 QWaylandDisplay::~QWaylandDisplay(void)
 {
+    if (m_eventThread)
+        m_eventThread->stop();
+
+    if (m_frameEventQueueThread)
+        m_frameEventQueueThread->stop();
+
     if (mSyncCallback)
         wl_callback_destroy(mSyncCallback);
 
@@ -211,100 +408,34 @@ void QWaylandDisplay::ensureScreen()
     Q_ASSERT(!QGuiApplication::screens().empty());
 }
 
-void QWaylandDisplay::checkError() const
+// Called in main thread, either from queued signal or directly.
+void QWaylandDisplay::flushRequests()
 {
-    int ecode = wl_display_get_error(mDisplay);
-    if ((ecode == EPIPE || ecode == ECONNRESET)) {
-        // special case this to provide a nicer error
-        qWarning("The Wayland connection broke. Did the Wayland compositor die?");
-    } else {
-        qWarning("The Wayland connection experienced a fatal error: %s", strerror(ecode));
-    }
-    _exit(1);
+    m_eventThread->readAndDispatchEvents();
 }
 
-void QWaylandDisplay::flushRequests()
+// We have to wait until we have an eventDispatcher before creating the eventThread,
+// otherwise forceRoundTrip() may block inside _events_read() because eventThread is
+// polling.
+void QWaylandDisplay::initEventThread()
 {
-    if (wl_display_prepare_read(mDisplay) == 0) {
-        wl_display_read_events(mDisplay);
-    }
-
-    if (wl_display_dispatch_pending(mDisplay) < 0)
-        checkError();
-
-    {
-        QReadLocker locker(&m_frameQueueLock);
-        for (const FrameQueue &q : mExternalQueues) {
-            QMutexLocker locker(q.mutex);
-            while (wl_display_prepare_read_queue(mDisplay, q.queue) != 0)
-                wl_display_dispatch_queue_pending(mDisplay, q.queue);
-            wl_display_read_events(mDisplay);
-            wl_display_dispatch_queue_pending(mDisplay, q.queue);
-        }
-    }
+    m_eventThread.reset(
+            new EventThread(mDisplay, /* default queue */ nullptr, EventThread::EmitToDispatch));
+    connect(m_eventThread.get(), &EventThread::needReadAndDispatch, this,
+            &QWaylandDisplay::flushRequests, Qt::QueuedConnection);
+    m_eventThread->start();
 
-    wl_display_flush(mDisplay);
+    // wl_display_disconnect() free this.
+    m_frameEventQueue = wl_display_create_queue(mDisplay);
+    m_frameEventQueueThread.reset(
+            new EventThread(mDisplay, m_frameEventQueue, EventThread::SelfDispatch));
+    m_frameEventQueueThread->start();
 }
 
 void QWaylandDisplay::blockingReadEvents()
 {
     if (wl_display_dispatch(mDisplay) < 0)
-        checkError();
-}
-
-void QWaylandDisplay::destroyFrameQueue(const QWaylandDisplay::FrameQueue &q)
-{
-    QWriteLocker locker(&m_frameQueueLock);
-    auto it = std::find_if(mExternalQueues.begin(),
-                           mExternalQueues.end(),
-                           [&q] (const QWaylandDisplay::FrameQueue &other){ return other.queue == q.queue; });
-    Q_ASSERT(it != mExternalQueues.end());
-    mExternalQueues.erase(it);
-    if (q.queue != nullptr)
-        wl_event_queue_destroy(q.queue);
-    delete q.mutex;
-}
-
-QWaylandDisplay::FrameQueue QWaylandDisplay::createFrameQueue()
-{
-    QWriteLocker locker(&m_frameQueueLock);
-    FrameQueue q{createEventQueue()};
-    mExternalQueues.append(q);
-    return q;
-}
-
-wl_event_queue *QWaylandDisplay::createEventQueue()
-{
-    return wl_display_create_queue(mDisplay);
-}
-
-void QWaylandDisplay::dispatchQueueWhile(wl_event_queue *queue, std::function<bool ()> condition, int timeout)
-{
-    if (!condition())
-        return;
-
-    QElapsedTimer timer;
-    timer.start();
-    struct pollfd pFd = qt_make_pollfd(wl_display_get_fd(mDisplay), POLLIN);
-    while (timeout == -1 || timer.elapsed() < timeout) {
-        while (wl_display_prepare_read_queue(mDisplay, queue) != 0)
-            wl_display_dispatch_queue_pending(mDisplay, queue);
-
-        wl_display_flush(mDisplay);
-
-        const int remaining = qMax(timeout - timer.elapsed(), 0ll);
-        const int pollTimeout = timeout == -1 ? -1 : remaining;
-        if (qt_poll_msecs(&pFd, 1, pollTimeout) > 0)
-            wl_display_read_events(mDisplay);
-        else
-            wl_display_cancel_read(mDisplay);
-
-        if (wl_display_dispatch_queue_pending(mDisplay, queue) < 0)
-            checkError();
-
-        if (!condition())
-            break;
-    }
+        checkWaylandError(mDisplay);
 }
 
 void QWaylandDisplay::checkTextInputProtocol()
@@ -748,4 +879,6 @@ QWaylandCursorTheme *QWaylandDisplay::loadCursorTheme(const QString &name, int p
 
 } // namespace QtWaylandClient
 
+#include "qwaylanddisplay.moc"
+
 QT_END_NAMESPACE