gl: add support for AV12

AV12 is an internally conceived format that is actually the
combination of NV12 and an alpha plane.

This format is to add to gstreamer's webM transparency support for
vp8 and vp9. To this end, two I420 streams are independently decoded
simultaneously for the actual content and the alpha plane respectively
and these are then combined into A420.

This patch adds GL conversion support so that it is possible to convert
from AV12 to RGBA for the purposes of rendering it on a display.

The reverse conversion is also supplied.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/1152>

4 files changed, 66 insertions, 3 deletions

diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c
index 203eb4c9c..cf5f159c6 100644
--- a/gst-libs/gst/gl/gstglcolorconvert.c
+++ b/gst-libs/gst/gl/gstglcolorconvert.c
@@ -333,6 +333,24 @@ static const struct shader_templ templ_SEMI_PLANAR_to_RGB =
     GST_GL_TEXTURE_TARGET_2D
   };
 
+static const gchar templ_AV12_to_RGB_BODY[] =
+    "vec4 rgba;\n"
+    "vec4 ayuv;\n"
+    /* FIXME: should get the sampling right... */
+    "ayuv.x=texture2D(Ytex, texcoord * tex_scale0).r;\n"
+    "ayuv.yz=texture2D(UVtex, texcoord * tex_scale1).rg;\n"
+    "ayuv.a=texture2D(Atex, texcoord * tex_scale2).r;\n"
+    "rgba.rgb = yuv_to_rgb (ayuv.xyz, offset, coeff1, coeff2, coeff3);\n"
+    "rgba.a = ayuv.a;\n" /* copy alpha as is */
+    "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
+
+static const struct shader_templ templ_AV12_to_RGB =
+  { NULL,
+    DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, UVtex, Atex;\n",
+    { glsl_func_yuv_to_rgb, NULL, },
+    GST_GL_TEXTURE_TARGET_2D
+  };
+
   /* RGB to NV12/NV21/NV16/NV61 conversion */
   /* NV12/NV16: u, v
      NV21/NV61: v, u */
@@ -354,6 +372,28 @@ static const struct shader_templ templ_RGB_to_SEMI_PLANAR_YUV =
     GST_GL_TEXTURE_TARGET_2D
   };
 
+static const gchar templ_RGB_to_AV12_BODY[] =
+    "vec4 texel, uv_texel;\n"
+    "vec4 ayuv;\n"
+    "texel = texture2D(tex, texcoord).%c%c%c%c;\n"
+    "uv_texel = texture2D(tex, texcoord * tex_scale0 * chroma_sampling).%c%c%c%c;\n"
+    "ayuv.x = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3).x;\n"
+    "ayuv.yz = rgb_to_yuv (uv_texel.rgb, offset, coeff1, coeff2, coeff3).yz;\n"
+    /* copy alpha as is */
+    "ayuv.a = texel.a;\n"
+    "gl_FragData[0] = vec4(ayuv.x, 0.0, 0.0, 1.0);\n"
+    "gl_FragData[1] = vec4(ayuv.y, ayuv.z, 0.0, 1.0);\n"
+    "gl_FragData[2] = vec4(ayuv.a, 0.0, 0.0, 1.0);\n";
+
+static const struct shader_templ templ_RGB_to_AV12 =
+  { NULL,
+    DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n"
+    "uniform vec2 chroma_sampling;\n",
+    { glsl_func_rgb_to_yuv, NULL, },
+    GST_GL_TEXTURE_TARGET_2D
+  };
+
+
 /* YUY2:r,g,a
    UYVY:a,b,r */
 static const gchar templ_YUY2_UYVY_to_RGB_BODY[] =
@@ -988,7 +1028,7 @@ _init_supported_formats (GstGLContext * context, gboolean output,
   if (!output || (!context || context->gl_vtable->DrawBuffers))
     _append_value_string_list (supported_formats, "GBRA", "GBR", "RGBP", "BGRP",
         "Y444", "I420", "YV12", "Y42B", "Y41B", "NV12", "NV21", "NV16", "NV61",
-        "A420", NULL);
+        "A420", "AV12", NULL);
 
   /* Requires reading from a RG/LA framebuffer... */
   if (!context || (USING_GLES3 (context) || USING_OPENGL (context)))
@@ -1676,6 +1716,7 @@ _get_n_textures (GstVideoFormat v_format)
     case GST_VIDEO_FORMAT_GBR:
     case GST_VIDEO_FORMAT_RGBP:
     case GST_VIDEO_FORMAT_BGRP:
+    case GST_VIDEO_FORMAT_AV12:
       return 3;
     case GST_VIDEO_FORMAT_GBRA:
     case GST_VIDEO_FORMAT_A420:
@@ -1981,6 +2022,17 @@ _YUV_to_RGB (GstGLColorConvert * convert)
         info->shader_tex_names[1] = "UVtex";
         break;
       }
+      case GST_VIDEO_FORMAT_AV12:
+      {
+        info->templ = &templ_AV12_to_RGB;
+        info->frag_body =
+            g_strdup_printf (templ_AV12_to_RGB_BODY,
+            pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
+        info->shader_tex_names[0] = "Ytex";
+        info->shader_tex_names[1] = "UVtex";
+        info->shader_tex_names[2] = "Atex";
+        break;
+      }
       case GST_VIDEO_FORMAT_NV21:
       case GST_VIDEO_FORMAT_NV61:
       {
@@ -2132,6 +2184,14 @@ _RGB_to_YUV (GstGLColorConvert * convert)
         info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f;
       }
       break;
+    case GST_VIDEO_FORMAT_AV12:
+      info->templ = &templ_RGB_to_AV12,
+          info->frag_body = g_strdup_printf (templ_RGB_to_AV12_BODY,
+          pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
+          pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
+      info->out_n_textures = 3;
+      info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f;
+      break;
     case GST_VIDEO_FORMAT_NV21:
     case GST_VIDEO_FORMAT_NV61:
       info->templ = &templ_RGB_to_SEMI_PLANAR_YUV,
diff --git a/gst-libs/gst/gl/gstglcolorconvert.h b/gst-libs/gst/gl/gstglcolorconvert.h
index 039a4c3a2..4512fe17c 100644
--- a/gst-libs/gst/gl/gstglcolorconvert.h
+++ b/gst-libs/gst/gl/gstglcolorconvert.h
@@ -103,7 +103,7 @@ struct _GstGLColorConvertClass
                                "xBGR, ARGB, ABGR, GBRA, GBR, RGBP, BGRP, Y444, I420, YV12, Y42B, " \
                                "Y41B, NV12, NV21, NV16, NV61, YUY2, UYVY, Y210, AYUV, " \
                                "VUYA, Y410, GRAY8, GRAY16_LE, GRAY16_BE, " \
-                               "RGB16, BGR16, ARGB64, A420 " \
+                               "RGB16, BGR16, ARGB64, A420, AV12" \
                                GST_GL_COLOR_CONVERT_EXT_FORMATS "}"
 
 /**
diff --git a/gst-libs/gst/gl/gstglformat.c b/gst-libs/gst/gl/gstglformat.c
index dc940018a..d1ce3f03a 100644
--- a/gst-libs/gst/gl/gstglformat.c
+++ b/gst-libs/gst/gl/gstglformat.c
@@ -194,6 +194,9 @@ gst_gl_format_from_video_info (GstGLContext * context,
     case GST_VIDEO_FORMAT_NV61:
       n_plane_components = plane == 0 ? 1 : 2;
       break;
+    case GST_VIDEO_FORMAT_AV12:
+      n_plane_components = (plane == 1) ? 2 : 1;
+      break;
     case GST_VIDEO_FORMAT_GRAY8:
     case GST_VIDEO_FORMAT_Y444:
     case GST_VIDEO_FORMAT_Y42B:
diff --git a/gst-libs/gst/gl/gstglmemory.h b/gst-libs/gst/gl/gstglmemory.h
index 33296839f..daea4d12a 100644
--- a/gst-libs/gst/gl/gstglmemory.h
+++ b/gst-libs/gst/gl/gstglmemory.h
@@ -64,7 +64,7 @@ GType gst_gl_memory_allocator_get_type(void);
 #define GST_GL_MEMORY_VIDEO_FORMATS_STR \
     "{ RGBA, BGRA, RGBx, BGRx, ARGB, ABGR, xRGB, xBGR, GBRA, GBR, RGBP, BGRP, RGB, BGR, RGB16, BGR16, " \
     "AYUV, VUYA, Y410, I420, YV12, NV12, NV21, NV16, NV61, YUY2, UYVY, Y210, Y41B, " \
-    "Y42B, Y444, GRAY8, GRAY16_LE, GRAY16_BE, ARGB64, A420" \
+    "Y42B, Y444, GRAY8, GRAY16_LE, GRAY16_BE, ARGB64, A420, AV12" \
     GST_GL_MEMORY_VIDEO_EXT_FORMATS "}"
 
 /**