QWaylandQuickItem: Set shader for external OES texture manually

The shader for external OES texture cannot go through the normal
pipeline and thus is required to be created manually.

Note: This e.g. fixes wayland-egl backend on NVIDIA, as the
EGLStreams implementation depends on external OES buffer type.

Pick-to: 6.0 6.1
Task-number: QTBUG-89960
Change-Id: I9ab92b72f9db019be94e69a64d3fdf652dd4ed45
Reviewed-by: Jaeyoon Jung <jaeyoon.jung@lge.com>
Reviewed-by: Eskil Abrahamsen Blomfeldt <eskil.abrahamsen-blomfeldt@qt.io>

5 files changed, 99 insertions, 14 deletions

diff --git a/src/compositor/CMakeLists.txt b/src/compositor/CMakeLists.txt
index b079c480..361adb9e 100644
--- a/src/compositor/CMakeLists.txt
+++ b/src/compositor/CMakeLists.txt
@@ -85,6 +85,7 @@ set(compositor_resource_files
     "shaders/surface_y_u_v.frag.qsb"
     "shaders/surface_y_uv.frag.qsb"
     "shaders/surface_y_xuxv.frag.qsb"
+    "shaders/surface_oes_external.frag"
 )
 
 qt_internal_add_resource(WaylandCompositor "compositor"
diff --git a/src/compositor/compositor_api/qwaylandquickitem.cpp b/src/compositor/compositor_api/qwaylandquickitem.cpp
index f8e83d22..a578fd68 100644
--- a/src/compositor/compositor_api/qwaylandquickitem.cpp
+++ b/src/compositor/compositor_api/qwaylandquickitem.cpp
@@ -56,12 +56,17 @@
 #include <QtQuick/QQuickWindow>
 #include <QtQuick/qsgtexture.h>
 
+#include <QtCore/QFile>
 #include <QtCore/QMutexLocker>
 #include <QtCore/QMutex>
 
 #include <wayland-server-core.h>
 #include <QThread>
 
+#if QT_CONFIG(opengl)
+#include <QtGui/private/qshaderdescription_p_p.h>
+#endif
+
 #ifndef GL_TEXTURE_EXTERNAL_OES
 #define GL_TEXTURE_EXTERNAL_OES 0x8D65
 #endif
@@ -102,7 +107,7 @@ static const struct {
     // BufferFormatEgl_EXTERNAL_OES
     {
         ":/qt-project.org/wayland/compositor/shaders/surface.vert.qsb",
-        ":/qt-project.org/wayland/compositor/shaders/surface_oes_external.frag.qsb",
+        ":/qt-project.org/wayland/compositor/shaders/surface_oes_external.frag",
         GL_TEXTURE_EXTERNAL_OES, 1, false,
         QSGMaterial::Blending,
         {}
@@ -140,7 +145,88 @@ QWaylandBufferMaterialShader::QWaylandBufferMaterialShader(QWaylandBufferRef::Bu
     : m_format(format)
 {
     setShaderFileName(VertexStage, QString::fromLatin1(bufferTypes[format].vertexShaderSourceFile));
-    setShaderFileName(FragmentStage, QString::fromLatin1(bufferTypes[format].fragmentShaderSourceFile));
+    auto fragmentShaderSourceFile = QString::fromLatin1(bufferTypes[format].fragmentShaderSourceFile);
+
+    if (format == QWaylandBufferRef::BufferFormatEgl_EXTERNAL_OES)
+        setupExternalOESShader(fragmentShaderSourceFile);
+    else
+        setShaderFileName(FragmentStage, fragmentShaderSourceFile);
+}
+
+void QWaylandBufferMaterialShader::setupExternalOESShader(const QString &shaderFilename)
+{
+#if QT_CONFIG(opengl)
+    QFile shaderFile(shaderFilename);
+    if (!shaderFile.open(QIODevice::ReadOnly)) {
+        qCWarning(qLcWaylandCompositor) << "Cannot find external OES shader file:" << shaderFilename;
+        return;
+    }
+    QByteArray FS = shaderFile.readAll();
+
+    static const char *FS_GLES_PREAMBLE =
+        "#extension GL_OES_EGL_image_external : require\n"
+        "precision highp float;\n";
+    static const char *FS_GL_PREAMBLE =
+            "#version 120\n"
+            "#extension GL_OES_EGL_image_external : require\n";
+    QByteArray fsGLES = FS_GLES_PREAMBLE + FS;
+    QByteArray fsGL = FS_GL_PREAMBLE + FS;
+
+    QShaderDescription desc;
+    QShaderDescriptionPrivate *descData = QShaderDescriptionPrivate::get(&desc);
+
+    QShaderDescription::InOutVariable texCoordInput;
+    texCoordInput.name = "v_texcoord";
+    texCoordInput.type = QShaderDescription::Vec2;
+    texCoordInput.location = 0;
+
+    descData->inVars = { texCoordInput };
+
+    QShaderDescription::InOutVariable fragColorOutput;
+    texCoordInput.name = "gl_FragColor";
+    texCoordInput.type = QShaderDescription::Vec4;
+    texCoordInput.location = 0;
+
+    descData->outVars = { fragColorOutput };
+
+    QShaderDescription::BlockVariable matrixBlockVar;
+    matrixBlockVar.name = "qt_Matrix";
+    matrixBlockVar.type = QShaderDescription::Mat4;
+    matrixBlockVar.offset = 0;
+    matrixBlockVar.size = 64;
+
+    QShaderDescription::BlockVariable opacityBlockVar;
+    opacityBlockVar.name = "qt_Opacity";
+    opacityBlockVar.type = QShaderDescription::Float;
+    opacityBlockVar.offset = 64;
+    opacityBlockVar.size = 4;
+
+    QShaderDescription::UniformBlock ubufStruct;
+    ubufStruct.blockName = "buf";
+    ubufStruct.structName = "ubuf";
+    ubufStruct.size = 64 + 4;
+    ubufStruct.binding = 0;
+    ubufStruct.members = { matrixBlockVar, opacityBlockVar };
+
+    descData->uniformBlocks = { ubufStruct };
+
+    QShaderDescription::InOutVariable samplerTex0;
+    samplerTex0.name = "tex0";
+    samplerTex0.type = QShaderDescription::SamplerExternalOES;
+    samplerTex0.binding = 1;
+
+    descData->combinedImageSamplers = { samplerTex0 };
+
+    QShader shaderPack;
+    shaderPack.setStage(QShader::FragmentStage);
+    shaderPack.setDescription(desc);
+    shaderPack.setShader(QShaderKey(QShader::GlslShader, QShaderVersion(100, QShaderVersion::GlslEs)), QShaderCode(fsGLES));
+    shaderPack.setShader(QShaderKey(QShader::GlslShader, QShaderVersion(120)), QShaderCode(fsGL));
+
+    setShader(FragmentStage, shaderPack);
+#else
+    Q_UNUSED(shaderFilename);
+#endif
 }
 
 bool QWaylandBufferMaterialShader::updateUniformData(RenderState &state, QSGMaterial *, QSGMaterial *)
diff --git a/src/compositor/compositor_api/qwaylandquickitem_p.h b/src/compositor/compositor_api/qwaylandquickitem_p.h
index 216b1c5e..34acdeab 100644
--- a/src/compositor/compositor_api/qwaylandquickitem_p.h
+++ b/src/compositor/compositor_api/qwaylandquickitem_p.h
@@ -64,6 +64,7 @@ public:
                            QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override;
     void updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
                             QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override;
+    void setupExternalOESShader(const QString &shaderFilename);
 
 private:
     const QWaylandBufferRef::BufferFormatEgl m_format;
diff --git a/src/compositor/shaders/compile b/src/compositor/shaders/compile
index e7981b1f..33c2f34d 100644
--- a/src/compositor/shaders/compile
+++ b/src/compositor/shaders/compile
@@ -5,6 +5,5 @@ qsb --glsl "100 es,120,150" -o surface_y_uv.frag.qsb surface_y_uv.frag
 qsb --glsl "100 es,120,150" -o surface_y_u_v.frag.qsb surface_y_u_v.frag
 qsb --glsl "100 es,120,150" -o surface_y_xuxv.frag.qsb surface_y_xuxv.frag
 
-# Does not compile for whatever reason. We may not support that externalOES
-# thing at run time either, so skip for now.
-# qsb --glsl "100 es,120,150" -o surface_oes_external.frag.qsb surface_oes_external.frag
+# Cannot be precompiled and is handled separately:
+# surface_oes_external.frag
diff --git a/src/compositor/shaders/surface_oes_external.frag b/src/compositor/shaders/surface_oes_external.frag
index 37ae3668..1cec856b 100644
--- a/src/compositor/shaders/surface_oes_external.frag
+++ b/src/compositor/shaders/surface_oes_external.frag
@@ -1,17 +1,15 @@
-#version 440
-#extension GL_OES_EGL_image_external_essl3 : require
+// This shader stump cannot be precompiled and is compiled at run-time.
+// Appropriate target preamble added when it is loaded.
 
-layout(location = 0) in vec2 v_texcoord;
-layout(location = 0) out vec4 fragColor;
-
-layout(std140, binding = 0) uniform buf {
+varying vec2 v_texcoord;
+struct buf {
     mat4 qt_Matrix;
     float qt_Opacity;
 };
-
-layout(binding = 1) uniform samplerExternalOES tex0;
+uniform buf ubuf;
+uniform samplerExternalOES tex0;
 
 void main()
 {
-    fragColor = qt_Opacity * texture(tex0, v_texcoord);
+    gl_FragColor = ubuf.qt_Opacity * texture2D(tex0, v_texcoord);
 }