Fix malloc overflow check warning

To fix this warning:

../src/cairo-malloc-private.h:83:32: warning: comparison is always false due to limited range of data type [-Wtype-limits]
   83 |   ((size) != 0 && (size_t) (a) >= SIZE_MAX / (size_t) (size) ? NULL : \
      |                                ^~

Create two new macros to do the overflow checks:
_cairo_addl_size_t_overflow and _cairo_mul_size_t_overflow. Implement
them using compiler builtins where available.

Update cairo-malloc-private to use these fuctions and add an overflow
test to test the functions.

1 files changed, 42 insertions, 14 deletions

diff --git a/src/cairo-malloc-private.h b/src/cairo-malloc-private.h
index 40314e1ce..0de52a561 100644
--- a/src/cairo-malloc-private.h
+++ b/src/cairo-malloc-private.h
@@ -79,9 +79,15 @@
  * case of malloc() failure or overflow.
  **/
 
-#define _cairo_malloc_ab(a, size) \
-  ((size) != 0 && (size_t) (a) >= SIZE_MAX / (size_t) (size) ? NULL : \
-   _cairo_malloc((size_t) (a) * (size_t) (size)))
+static cairo_always_inline void *
+_cairo_malloc_ab(size_t a, size_t size)
+{
+    size_t c;
+    if (_cairo_mul_size_t_overflow (a, size, &c))
+	return NULL;
+
+    return _cairo_malloc(c);
+}
 
 /**
  * _cairo_realloc_ab:
@@ -101,9 +107,15 @@
  * of memory * is left untouched).
  **/
 
-#define _cairo_realloc_ab(ptr, a, size) \
-  ((size) != 0 && (size_t) (a) >= SIZE_MAX / (size_t) (size) ? NULL : \
-   realloc(ptr, (size_t) (a) * (size_t) (size)))
+static cairo_always_inline void *
+_cairo_realloc_ab(void *ptr, size_t a, size_t size)
+{
+    size_t c;
+    if (_cairo_mul_size_t_overflow (a, size, &c))
+	return NULL;
+
+    return realloc(ptr, c);
+}
 
 /**
  * _cairo_malloc_abc:
@@ -122,10 +134,18 @@
  * case of malloc() failure or overflow.
  **/
 
-#define _cairo_malloc_abc(a, b, size) \
-  ((b) != 0 && (size_t) (a) >= SIZE_MAX / (size_t) (b) ? NULL : \
-   (size) != 0 && (size_t) ((a)*(b)) >= SIZE_MAX / (size_t) (size) ? NULL : \
-   _cairo_malloc((size_t) (a) * (size_t) (b) * (size_t) (size)))
+static cairo_always_inline void *
+_cairo_malloc_abc(size_t a, size_t b, size_t size)
+{
+    size_t c, d;
+    if (_cairo_mul_size_t_overflow (a, b, &c))
+	return NULL;
+
+    if (_cairo_mul_size_t_overflow (c, size, &d))
+	return NULL;
+
+    return _cairo_malloc(d);
+}
 
 /**
  * _cairo_malloc_ab_plus_c:
@@ -141,9 +161,17 @@
  * case of malloc() failure or overflow.
  **/
 
-#define _cairo_malloc_ab_plus_c(a, size, c) \
-  ((size) != 0 && (size_t) (a) >= SIZE_MAX / (size_t) (size) ? NULL : \
-   (size_t) (c) >= SIZE_MAX - (size_t) (a) * (size_t) (size) ? NULL : \
-   _cairo_malloc((size_t) (a) * (size_t) (size) + (size_t) (c)))
+static cairo_always_inline void *
+_cairo_malloc_ab_plus_c(size_t a, size_t size, size_t c)
+{
+    size_t d, e;
+    if (_cairo_mul_size_t_overflow (a, size, &d))
+	return NULL;
+
+    if (_cairo_add_size_t_overflow (d, c, &e))
+	return NULL;
+
+    return _cairo_malloc(e);
+}
 
 #endif /* CAIRO_MALLOC_PRIVATE_H */