image: Don't crash on weird pixman formats

_pixel_to_solid() used to assert that it got a known cairo_format_t.
However, this might not be the case when backends decide to use a pixman
format that is not representable by a cairo format (X and DirectFB are
examples for backends that do that).

This patch makes _pixel_to_solid() return NULL in that case and fixes
the callers to deal with it.

https://bugs.freedesktop.org/show_bug.cgi?id=37916

1 files changed, 16 insertions, 7 deletions

diff --git a/src/cairo-image-surface.c b/src/cairo-image-surface.c
index 7c56f1680..ceffbc4c3 100644
--- a/src/cairo-image-surface.c
+++ b/src/cairo-image-surface.c
@@ -1316,10 +1316,12 @@ _pixel_to_solid (cairo_image_surface_t *image, int x, int y)
 
     switch (image->format) {
     default:
-    case CAIRO_FORMAT_INVALID:
 	ASSERT_NOT_REACHED;
 	return NULL;
 
+    case CAIRO_FORMAT_INVALID:
+	return NULL;
+
     case CAIRO_FORMAT_A1:
 	pixel = *(uint8_t *) (image->data + y * image->stride + x/8);
 	return pixel & (1 << (x&7)) ? _pixman_white_image () : _pixman_transparent_image ();
@@ -1415,7 +1417,9 @@ _pixman_image_for_surface (const cairo_surface_pattern_t *pattern,
 		}
 		else
 		{
-		    return _pixel_to_solid (source, sample.x, sample.y);
+		    pixman_image = _pixel_to_solid (source, sample.x, sample.y);
+                    if (pixman_image)
+                        return pixman_image;
 		}
 	    }
 
@@ -1453,9 +1457,11 @@ _pixman_image_for_surface (const cairo_surface_pattern_t *pattern,
 
 	    if (sample.width == 1 && sample.height == 1) {
 		if (is_contained) {
-		    return _pixel_to_solid (source,
-					    sub->extents.x + sample.x,
-					    sub->extents.y + sample.y);
+		    pixman_image = _pixel_to_solid (source,
+                                                    sub->extents.x + sample.x,
+                                                    sub->extents.y + sample.y);
+                    if (pixman_image)
+                        return pixman_image;
 		} else {
 		    if (extend == CAIRO_EXTEND_NONE)
 			return _pixman_transparent_image ();
@@ -1509,8 +1515,11 @@ _pixman_image_for_surface (const cairo_surface_pattern_t *pattern,
 	    else
 	    {
 		pixman_image = _pixel_to_solid (image, sample.x, sample.y);
-		_cairo_surface_release_source_image (pattern->surface, image, extra);
-		return pixman_image;
+                if (pixman_image)
+                {
+                    _cairo_surface_release_source_image (pattern->surface, image, extra);
+                    return pixman_image;
+                }
 	    }
 	}