Make x11grab cursor drawing suck less

This new version:
1.  Works on 24-bit and 32-bit input, not just 32-bit.
2.  Doesn't try to run on 16-bit or 8-bit, instead of outright crashing.
3.  Does proper alpha-blending, so cursor shadows look correct.
4.  Doesn't swap R and B.

Mostly fixes issue 1997.
Fixes issue 2056.

Originally committed as revision 25690 to svn://svn.ffmpeg.org/ffmpeg/trunk

1 files changed, 26 insertions, 9 deletions

diff --git a/libavdevice/x11grab.c b/libavdevice/x11grab.c
index 1b58d44f2e..ee3b8e901e 100644
--- a/libavdevice/x11grab.c
+++ b/libavdevice/x11grab.c
@@ -256,7 +256,16 @@ paint_mouse_pointer(XImage *image, struct x11_grab *s)
     int x, y;
     int line, column;
     int to_line, to_column;
-    int image_addr, xcim_addr;
+    int pixstride = image->bits_per_pixel >> 3;
+    /* Warning: in its insanity, xlib provides unsigned image data through a
+     * char* pointer, so we have to make it uint8_t to make things not break.
+     * Anyone who performs further investigation of the xlib API likely risks
+     * permanent brain damage. */
+    uint8_t *pix = image->data;
+
+    /* Code doesn't currently support 16-bit or PAL8 */
+    if (image->bits_per_pixel != 24 && image->bits_per_pixel != 32)
+        return;
 
     xcim = XFixesGetCursorImage(dpy);
 
@@ -268,14 +277,22 @@ paint_mouse_pointer(XImage *image, struct x11_grab *s)
 
     for (line = FFMAX(y, y_off); line < to_line; line++) {
         for (column = FFMAX(x, x_off); column < to_column; column++) {
-            xcim_addr = (line - y) * xcim->width + column - x;
-
-            if ((unsigned char)(xcim->pixels[xcim_addr] >> 24) != 0) { // skip fully transparent pixel
-                image_addr = ((line - y_off) * width + column - x_off) * 4;
-
-                image->data[image_addr] = (unsigned char)(xcim->pixels[xcim_addr] >> 0);
-                image->data[image_addr+1] = (unsigned char)(xcim->pixels[xcim_addr] >> 8);
-                image->data[image_addr+2] = (unsigned char)(xcim->pixels[xcim_addr] >> 16);
+            int  xcim_addr = (line - y) * xcim->width + column - x;
+            int image_addr = ((line - y_off) * width + column - x_off) * pixstride;
+            int r = (uint8_t)(xcim->pixels[xcim_addr] >>  0);
+            int g = (uint8_t)(xcim->pixels[xcim_addr] >>  8);
+            int b = (uint8_t)(xcim->pixels[xcim_addr] >> 16);
+            int a = (uint8_t)(xcim->pixels[xcim_addr] >> 24);
+
+            if (a == 255) {
+                pix[image_addr+0] = r;
+                pix[image_addr+1] = g;
+                pix[image_addr+2] = b;
+            } else if (a) {
+                /* pixel values from XFixesGetCursorImage come premultiplied by alpha */
+                pix[image_addr+0] = r + (pix[image_addr+0]*(255-a) + 255/2) / 255;
+                pix[image_addr+1] = g + (pix[image_addr+1]*(255-a) + 255/2) / 255;
+                pix[image_addr+2] = b + (pix[image_addr+2]*(255-a) + 255/2) / 255;
             }
         }
     }