Add support for BGRA glyphs display and scaling

Display is done using an XRender Picture, as XRender
glyphs are incompatible with BGRA rendering due to
their use of the glyph bitmap as a mask.

Scaling is done by averaging all relevant pixel, which gives
much better result than nearest pixel sampling while staying
simple enough and not too computationally expensive.

This enables color emoji rendering support.

Fixes: #6

Signed-off-by: Maxime Coste <mawww@kakoune.org>
Signed-off-by: Thomas E. Dickey <dickey@invisible-island.net>

4 files changed, 293 insertions, 30 deletions

diff --git a/src/xftfreetype.c b/src/xftfreetype.c
index 4bba26b..51d4613 100644
--- a/src/xftfreetype.c
+++ b/src/xftfreetype.c
@@ -524,7 +524,7 @@ XftFontInfoFill (Display *dpy, _Xconst FcPattern *pattern, XftFontInfo *fi)
     /*
      * Compute glyph load flags
      */
-    fi->load_flags = FT_LOAD_DEFAULT;
+    fi->load_flags = FT_LOAD_DEFAULT | FT_LOAD_COLOR;
 
 #ifndef XFT_EMBEDDED_BITMAP
 #define XFT_EMBEDDED_BITMAP "embeddedbitmap"
@@ -776,6 +776,7 @@ XftFontOpenInfo (Display	*dpy,
     FcChar32		hash_value;
     FcChar32		rehash_value;
     FcBool		antialias;
+    FcBool		color;
     int			max_glyph_memory;
     size_t		alloc_size;
     int			ascent, descent, height;
@@ -832,12 +833,18 @@ XftFontOpenInfo (Display	*dpy,
     if (!(face->face_flags & FT_FACE_FLAG_SCALABLE))
 	antialias = FcFalse;
 
+    color = FT_HAS_COLOR(face) ? FcTrue : FcFalse;
+
     /*
      * Find the appropriate picture format
      */
     if (fi->render)
     {
-	if (antialias)
+	if (color)
+	{
+	    format = XRenderFindStandardFormat (dpy, PictStandardARGB32);
+	}
+	else if (antialias)
 	{
 	    switch (fi->rgba) {
 	    case FC_RGBA_RGB:
@@ -968,6 +975,13 @@ XftFontOpenInfo (Display	*dpy,
      * which doesn't happen in XftFontInfoFill
      */
     font->info.antialias = antialias;
+
+    /*
+     * Set color value, which is only known once the
+     * font was loaded
+     */
+    font->info.color = color;
+
     /*
      * bump XftFile reference count
      */
diff --git a/src/xftglyphs.c b/src/xftglyphs.c
index b27962b..df167f3 100644
--- a/src/xftglyphs.c
+++ b/src/xftglyphs.c
@@ -27,6 +27,8 @@
 
 #include FT_SYNTHESIS_H
 
+#include FT_GLYPH_H
+
 /*
  * Validate the memory info for a font
  */
@@ -141,9 +143,11 @@ _XftValidateGlyphUsage(XftFontInt *font)
 static int
 _compute_xrender_bitmap_size( FT_Bitmap*	target,
 			      FT_GlyphSlot	slot,
-			      FT_Render_Mode	mode )
+			      FT_Render_Mode	mode,
+			      FT_Matrix*        matrix )
 {
     FT_Bitmap*	ftbit;
+    FT_Vector	vector;
     int		width, height, pitch;
 
     if ( slot->format != FT_GLYPH_FORMAT_BITMAP )
@@ -154,6 +158,16 @@ _compute_xrender_bitmap_size( FT_Bitmap*	target,
 
     width = (int)ftbit->width;
     height = (int)ftbit->rows;
+
+    if ( matrix && mode == FT_RENDER_MODE_NORMAL )
+    {
+	vector.x = ftbit->width;
+	vector.y = ftbit->rows;
+	FT_Vector_Transform(&vector, matrix);
+
+	width = (int)vector.x;
+	height = (int)vector.y;
+    }
     pitch = (width+3) & ~3;
 
     switch ( ftbit->pixel_mode )
@@ -175,6 +189,10 @@ _compute_xrender_bitmap_size( FT_Bitmap*	target,
 	}
 	break;
 
+    case FT_PIXEL_MODE_BGRA:
+	pitch = width * 4;
+	break;
+
     case FT_PIXEL_MODE_LCD:
 	if ( mode != FT_RENDER_MODE_LCD )
 	    return -1;
@@ -206,6 +224,105 @@ _compute_xrender_bitmap_size( FT_Bitmap*	target,
 }
 
 /* this functions converts the glyph bitmap found in a FT_GlyphSlot
+ * into a different format while scaling by applying the given matrix
+ * (see _compute_xrender_bitmap_size)
+ *
+ * you should call this function after _compute_xrender_bitmap_size
+ *
+ * target :: target bitmap descriptor. Note that its 'buffer' pointer
+ *           must point to memory allocated by the caller
+ *
+ * source :: the source bitmap descriptor
+ *
+ * matrix :: the scaling matrix to apply
+ */
+static void
+_scaled_fill_xrender_bitmap( FT_Bitmap*	target,
+		             FT_Bitmap* source,
+                             const FT_Matrix* matrix )
+{
+    unsigned char*	src_buf	  = source->buffer;
+    unsigned char*	dst_line  = target->buffer;
+    int			src_pitch = source->pitch;
+    int			width     = target->width;
+    int			height    = target->rows;
+    int			pitch     = target->pitch;
+    int			h;
+    FT_Vector		vector;
+    FT_Matrix		inverse	  = *matrix;
+    int			sampling_width;
+    int			sampling_height;
+    int			sample_count;
+
+    if ( src_pitch < 0 )
+	src_buf -= src_pitch * (source->rows - 1);
+
+    FT_Matrix_Invert(&inverse);
+
+    /* compute how many source pixels a target pixel spans */
+    vector.x = 1;
+    vector.y = 1;
+    FT_Vector_Transform(&vector, &inverse);
+    sampling_width = vector.x / 2;
+    sampling_height = vector.y / 2;
+    sample_count = (2 * sampling_width + 1) * (2 * sampling_height + 1);
+
+    for	( h = height; h	> 0; h--, dst_line += pitch )
+    {
+	int x;
+
+	for ( x	= 0; x < width;	x++ )
+	{
+	    unsigned char* src;
+
+#define CLAMP(x, min, max) ((x) < (min) ? (min) : ((x) > (max) ? (max) : (x)))
+
+            /* compute target pixel location in source space */
+	    vector.x = (x            * 0x10000) + 0x10000 / 2;
+	    vector.y = ((height - h) * 0x10000) + 0x10000 / 2;
+	    FT_Vector_Transform(&vector, &inverse);
+	    vector.x = CLAMP(FT_RoundFix(vector.x) / 0x10000, 0, source->width - 1);
+	    vector.y = CLAMP(FT_RoundFix(vector.y) / 0x10000, 0, source->rows  - 1);
+
+	    switch ( source->pixel_mode	)
+	    {
+	    case FT_PIXEL_MODE_MONO: /* convert mono to 8-bit gray, scale using nearest pixel */
+		src = src_buf + (vector.y * src_pitch);
+		if ( src[(vector.x >> 3)] & (0x80 >> (vector.x & 7)) )
+		    dst_line[x] = 0xff;
+		break;
+
+	    case FT_PIXEL_MODE_GRAY: /* scale using nearest pixel */
+		src = src_buf + (vector.y * src_pitch);
+		dst_line[x] = src[vector.x];
+		break;
+
+	    case FT_PIXEL_MODE_BGRA: /* scale by averaging all relevant source pixels, keep BGRA format */
+	    {
+		int sample_x, sample_y;
+		int bgra[4] = {};
+		for (sample_y = - sampling_height; sample_y < sampling_height + 1; ++sample_y)
+		{
+		    int src_y = CLAMP(vector.y + sample_y, 0, source->rows - 1);
+		    src = src_buf + (src_y * src_pitch);
+		    for (sample_x = - sampling_width; sample_x < sampling_width + 1; ++sample_x)
+		    {
+			int src_x = CLAMP(vector.x + sample_x, 0, source->width - 1);
+			for (int i = 0; i < 4; ++i)
+			    bgra[i] += src[src_x * 4 + i];
+		    }
+		}
+
+		for (int i = 0; i < 4; ++i)
+		    dst_line[4 * x + i] = bgra[i] / sample_count;
+		break;
+	    }
+	    }
+	}
+    }
+}
+
+/* this functions converts the glyph bitmap found in a FT_GlyphSlot
  * into a different format (see _compute_xrender_bitmap_size)
  *
  * you should call this function after _compute_xrender_bitmap_size
@@ -307,6 +424,11 @@ _fill_xrender_bitmap( FT_Bitmap*	target,
 	    }
 	    break;
 
+	case FT_PIXEL_MODE_BGRA: /* Preserve BGRA format */
+	    for ( h = height; h > 0; h--, srcLine += src_pitch, dstLine += pitch )
+		memcpy( dstLine, srcLine, width * 4 );
+	    break;
+
 	case FT_PIXEL_MODE_LCD:
 	    if ( !bgr )
 	    {
@@ -402,6 +524,55 @@ _fill_xrender_bitmap( FT_Bitmap*	target,
     }
 }
 
+/* This function creates a Picture for the given glyph on the default root window
+ * It will only work in Xinerama mode
+ * 
+ * dpy    :: target display
+ * 
+ * format :: target pixmap format
+ * 
+ * width  :: picture width
+ * 
+ * width  :: picture height
+ * 
+ * data   :: bitmap data
+ * 
+ */
+static Picture
+_create_glyph_bgra_picture (Display           *dpy,
+                            XRenderPictFormat *format,
+                            int               width,
+                            int               height,
+                            unsigned char     *data)
+{
+    XImage image = {
+        width, height, 0, ZPixmap, (char *)data,
+        dpy->byte_order, dpy->bitmap_unit, dpy->bitmap_bit_order, 32,
+        32, 0, 32,
+        0, 0, 0
+    };
+    Picture picture;
+    Pixmap pixmap;
+    GC gc;
+
+    pixmap = XCreatePixmap(dpy, DefaultRootWindow(dpy), width, height, 32);
+    if (!pixmap)
+        return None;
+
+    gc = XCreateGC(dpy, pixmap, 0, NULL);
+    if (!gc)
+        return None;
+
+    XInitImage(&image);
+    XPutImage(dpy, pixmap, gc, &image, 0, 0, 0, 0, width, height);
+    picture = XRenderCreatePicture(dpy, pixmap, format, 0, NULL);
+
+    XFreeGC(dpy, gc);
+    XFreePixmap(dpy, pixmap);
+
+    return picture;
+}
+
 _X_EXPORT void
 XftFontLoadGlyphs (Display	    *dpy,
 		   XftFont	    *pub,
@@ -428,6 +599,8 @@ XftFontLoadGlyphs (Display	    *dpy,
     FT_Vector	    vector;
     FT_Face	    face;
     FT_Render_Mode  mode = FT_RENDER_MODE_MONO;
+    FcBool	    transform;
+    FcBool	    glyph_transform;
 
     if (!info)
 	return;
@@ -437,6 +610,8 @@ XftFontLoadGlyphs (Display	    *dpy,
     if (!face)
 	return;
 
+    if (font->info.color)
+        mode = FT_RENDER_MODE_NORMAL;
     if (font->info.antialias)
     {
 	switch (font->info.rgba) {
@@ -453,6 +628,8 @@ XftFontLoadGlyphs (Display	    *dpy,
 	}
     }
 
+    transform = font->info.transform && mode != FT_RENDER_MODE_MONO;
+
     while (nglyph--)
     {
 	glyphindex = *glyphs++;
@@ -503,7 +680,7 @@ XftFontLoadGlyphs (Display	    *dpy,
 	/*
 	 * Compute glyph metrics from FreeType information
 	 */
-	if(font->info.transform && glyphslot->format != FT_GLYPH_FORMAT_BITMAP)
+	if (transform)
 	{
 	    /*
 	     * calculate the true width by transforming all four corners.
@@ -547,7 +724,7 @@ XftFontLoadGlyphs (Display	    *dpy,
 	 * Clip charcell glyphs to the bounding box
 	 * XXX transformed?
 	 */
-	if (font->info.spacing >= FC_CHARCELL && !font->info.transform)
+	if (font->info.spacing >= FC_CHARCELL && !transform)
 	{
 	    if (font->info.load_flags & FT_LOAD_VERTICAL_LAYOUT)
 	    {
@@ -577,18 +754,20 @@ XftFontLoadGlyphs (Display	    *dpy,
 	    }
 	}
 
+	glyph_transform = transform;
 	if ( glyphslot->format != FT_GLYPH_FORMAT_BITMAP )
 	{
 	    error = FT_Render_Glyph( face->glyph, mode );
 	    if (error)
 		continue;
+	    glyph_transform = False;
 	}
 
 	FT_Library_SetLcdFilter( _XftFTlibrary, FT_LCD_FILTER_NONE );
 
 	if (font->info.spacing >= FC_MONO)
 	{
-	    if (font->info.transform)
+	    if (transform)
 	    {
 		if (font->info.load_flags & FT_LOAD_VERTICAL_LAYOUT)
 		{
@@ -671,14 +850,27 @@ XftFontLoadGlyphs (Display	    *dpy,
 	    }
 	}
 
-	size = _compute_xrender_bitmap_size( &local, glyphslot, mode );
+	size = _compute_xrender_bitmap_size( &local, glyphslot, mode, glyph_transform ? &font->info.matrix : NULL );
 	if ( size < 0 )
 	    continue;
 
 	xftg->metrics.width  = (unsigned short)local.width;
 	xftg->metrics.height = (unsigned short)local.rows;
-	xftg->metrics.x      = (short)(- glyphslot->bitmap_left);
-	xftg->metrics.y      = (short)(  glyphslot->bitmap_top);
+	if (transform)
+	{
+	    vector.x = - glyphslot->bitmap_left;
+	    vector.y =   glyphslot->bitmap_top;
+
+	    FT_Vector_Transform(&vector, &font->info.matrix);
+
+	    xftg->metrics.x = (short)vector.x;
+	    xftg->metrics.y = (short)vector.y;
+	}
+	else
+	{
+	    xftg->metrics.x = (short)(- glyphslot->bitmap_left);
+	    xftg->metrics.y = (short)(  glyphslot->bitmap_top);
+	}
 
 	/*
 	 * If the glyph is relatively large (> 1% of server memory),
@@ -703,9 +895,12 @@ XftFontLoadGlyphs (Display	    *dpy,
 
 	local.buffer = bufBitmap;
 
-	_fill_xrender_bitmap( &local, glyphslot, mode,
-			      (font->info.rgba == FC_RGBA_BGR ||
-			       font->info.rgba == FC_RGBA_VBGR ) );
+        if (mode == FT_RENDER_MODE_NORMAL && glyph_transform)
+            _scaled_fill_xrender_bitmap(&local, &glyphslot->bitmap, &font->info.matrix);
+        else
+	    _fill_xrender_bitmap( &local, glyphslot, mode,
+			          (font->info.rgba == FC_RGBA_BGR ||
+			           font->info.rgba == FC_RGBA_VBGR ) );
 
 	/*
 	 * Copy or convert into local buffer.
@@ -720,6 +915,7 @@ XftFontLoadGlyphs (Display	    *dpy,
 	 */
 	glyph = (Glyph) glyphindex;
 
+	xftg->picture = 0;
 	xftg->glyph_memory = (size_t)size + font->sizeof_glyph;
 	if (font->format)
 	{
@@ -743,15 +939,21 @@ XftFontLoadGlyphs (Display	    *dpy,
 		    }
 		}
 	    }
-	    else if ( mode != FT_RENDER_MODE_NORMAL )
+	    else if (glyphslot->bitmap.pixel_mode == FT_PIXEL_MODE_BGRA || mode != FT_RENDER_MODE_NORMAL)
 	    {
 		/* invert ARGB <=> BGRA */
 		if (ImageByteOrder (dpy) != XftNativeByteOrder ())
 		    XftSwapCARD32 ((CARD32 *) bufBitmap, size >> 2);
 	    }
-	    XRenderAddGlyphs (dpy, font->glyphset, &glyph,
-			      &xftg->metrics, 1,
-			      (char *) bufBitmap, size);
+
+	    if (glyphslot->bitmap.pixel_mode == FT_PIXEL_MODE_BGRA)
+    	        xftg->picture = _create_glyph_bgra_picture(dpy, font->format,
+    	                                                   local.width, local.rows,
+    	                                                   bufBitmap);
+	    else
+		XRenderAddGlyphs (dpy, font->glyphset, &glyph,
+				  &xftg->metrics, 1,
+				  (char *) bufBitmap, size);
 	}
 	else
 	{
@@ -843,7 +1045,9 @@ XftFontUnloadGlyphs (Display		*dpy,
 			glyphindex, xftg->glyph_memory);
 	    if (font->format)
 	    {
-		if (font->glyphset)
+		if (xftg->picture)
+		    XRenderFreePicture(dpy, xftg->picture);
+		else if (font->glyphset)
 		{
 		    glyphBuf[nused++] = (Glyph) glyphindex;
 		    if (nused == sizeof (glyphBuf) / sizeof (glyphBuf[0]))
diff --git a/src/xftint.h b/src/xftint.h
index e1a792d..c59985a 100644
--- a/src/xftint.h
+++ b/src/xftint.h
@@ -87,6 +87,7 @@ typedef struct _XftGlyph {
     XGlyphInfo	    metrics;
     void	    *bitmap;
     unsigned long   glyph_memory;
+    Picture         picture;
 } XftGlyph;
 
 /*
@@ -146,6 +147,7 @@ struct _XftFontInfo {
     FT_F26Dot6		xsize, ysize;	/* pixel size */
     FcBool		antialias;	/* doing antialiasing */
     FcBool		embolden;	/* force emboldening */
+    FcBool		color;		/* contains color glyphs */
     int			rgba;		/* subpixel order */
     int			lcd_filter;	/* lcd filter */
     FT_Matrix		matrix;		/* glyph transformation matrix */
diff --git a/src/xftrender.c b/src/xftrender.c
index aad5431..721cae0 100644
--- a/src/xftrender.c
+++ b/src/xftrender.c
@@ -84,12 +84,14 @@ XftGlyphRender (Display		*dpy,
 		int		nglyphs)
 {
     XftFontInt	    *font = (XftFontInt *) pub;
-    int		    i;
+    int		    i, j;
     FT_UInt	    missing[XFT_NMISSING];
     int		    nmissing;
     FT_UInt	    g, max;
     int		    size, width;
+    int		    dstx, dsty;
     Glyph	    wire;
+    XftGlyph*       glyph;
     char	    *char8;
     unsigned short  *char16;
     unsigned int    *char32;
@@ -141,22 +143,46 @@ XftGlyphRender (Display		*dpy,
 	if (!chars)
 	    goto bail1;
     }
+    dstx = x;
+    dsty = y;
     char8 = (char *) chars;
     char16 = (unsigned short *) chars;
     char32 = (unsigned int *) chars;
-    for (i = 0; i < nglyphs; i++)
+    for (i = 0, j = 0; i < nglyphs; i++)
     {
 	wire = (Glyph) glyphs[i];
 	if (wire >= (Glyph) font->num_glyphs || !font->glyphs[wire])
 	    wire = 0;
-	switch (width) {
-	case 1: char8[i] = (char) wire; break;
-	case 2: char16[i] = (unsigned short) wire; break;
-	case 4: char32[i] = (unsigned int) wire; break;
+        glyph = font->glyphs[wire];
+	if (glyph->picture)
+	{
+	    _XftCompositeString(dpy, op, src, dst, font->format, font->glyphset,
+	                        srcx, srcy, x, y, width, chars, j);
+	    XRenderComposite(dpy, PictOpOver, glyph->picture, None,
+	                     dst, 0, 0, 0, 0, dstx, dsty - glyph->metrics.y,
+	                     glyph->metrics.width, glyph->metrics.height);
+
+	    dstx += glyph->metrics.xOff;
+	    dsty += glyph->metrics.yOff;
+
+	    x = dstx;
+	    y = dsty;
+	    j = 0;
+	}
+	else
+	{
+	    switch (width) {
+	    case 1: char8[j] = (char) wire; break;
+	    case 2: char16[j] = (unsigned short) wire; break;
+	    case 4: char32[j] = (unsigned int) wire; break;
+	    }
+	    dstx += glyph->metrics.xOff;
+	    dsty += glyph->metrics.yOff;
+	    ++j;
 	}
     }
     _XftCompositeString(dpy, op, src, dst, font->format, font->glyphset,
-                        srcx, srcy, x, y, width, chars, nglyphs);
+                        srcx, srcy, x, y, width, chars, j);
     if (chars != char_local)
 	free (chars);
 bail1:
@@ -319,9 +345,10 @@ XftGlyphSpecRender (Display		    *dpy,
 	    g = 0;
 	/*
 	 * check to see if the glyph is placed where it would
-	 * fall using the normal spacing
+	 * fall using the normal spacing and if it would render
+	 * as a XRender glyph
 	 */
-	if ((glyph = font->glyphs[g]))
+	if ((glyph = font->glyphs[g]) && !glyph->picture)
 	{
 	    if (x != glyphs[i].x || y != glyphs[i].y)
 	    {
@@ -335,7 +362,7 @@ XftGlyphSpecRender (Display		    *dpy,
     }
 
     elts = elts_local;
-    if (nelt > NUM_ELT_LOCAL)
+    if (!font->info.color && nelt > NUM_ELT_LOCAL)
     {
 	elts = AllocGlyphElt8Array (nelt);
 	if (!elts)
@@ -343,7 +370,7 @@ XftGlyphSpecRender (Display		    *dpy,
     }
 
     /*
-     * Generate the list of glyph elts
+     * Generate the list of glyph elts or render color glyphs
      */
     nelt = 0;
     x = y = 0;
@@ -357,6 +384,14 @@ XftGlyphSpecRender (Display		    *dpy,
 	    g = 0;
 	if ((glyph = font->glyphs[g]))
 	{
+	    if (glyph->picture)
+	    {
+                XRenderComposite(dpy, PictOpOver, glyph->picture, None,
+                                 dst, 0, 0, 0, 0,
+                                 glyphs[i].x, glyphs[i].y - glyph->metrics.y,
+                                 glyph->metrics.width, glyph->metrics.height);
+                continue;
+	    }
 	    if (!i || x != glyphs[i].x || y != glyphs[i].y)
 	    {
 		if (n)
@@ -592,7 +627,7 @@ XftGlyphFontSpecRender (Display			    *dpy,
 	 * check to see if the glyph is placed where it would
 	 * fall using the normal spacing
 	 */
-	if ((glyph = font->glyphs[g]))
+	if ((glyph = font->glyphs[g]) && !glyph->picture)
 	{
 	    if (pub != prevPublic || x != glyphs[i].x || y != glyphs[i].y)
 	    {
@@ -617,7 +652,7 @@ XftGlyphFontSpecRender (Display			    *dpy,
     }
 
     /*
-     * Generate the list of glyph elts
+     * Generate the list of glyph elts and render color glyphs
      */
     nelt = 0;
     x = y = 0;
@@ -635,6 +670,14 @@ XftGlyphFontSpecRender (Display			    *dpy,
 	    g = 0;
 	if ((glyph = font->glyphs[g]))
 	{
+	    if (glyph->picture)
+	    {
+                XRenderComposite(dpy, PictOpOver, glyph->picture, None,
+                                 dst, 0, 0, 0, 0,
+                                 glyphs[i].x, glyphs[i].y - glyph->metrics.y,
+                                 glyph->metrics.width, glyph->metrics.height);
+                continue;
+	    }
 	    if (!i || pub != prevPublic || x != glyphs[i].x || y != glyphs[i].y)
 	    {
 		if (n)