Add XtReallocArray() for overflow checking of multiplied args

Uses reallocarray() if available, otherwise checks for overflow itself,
if overflow is possible (i.e. in ILP32 & ILP64 environments, but not LP64
with 32-bit ints).

Includes unit tests and XtMallocArray() helper macro.

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

2 files changed, 169 insertions, 1 deletions

diff --git a/test/Alloc.c b/test/Alloc.c
index 3120f0f..03f44a8 100644
--- a/test/Alloc.c
+++ b/test/Alloc.c
@@ -569,6 +569,166 @@ static void test_XtRealloc_overflow(void)
 }
 
 
+/* Make sure XtReallocArray() works for a non-zero amount of memory */
+static void test_XtReallocArray_normal(void)
+{
+    void *p, *p2;
+    char *p3;
+
+    errno = 0;
+
+    /* Make sure reallocarray with a NULL pointer acts as malloc */
+    p = XtReallocArray(NULL, 8, 14);
+    g_assert_nonnull(p);
+    CHECK_ALIGNMENT(p);
+    CHECK_SIZE(p, 8 * 14);
+
+    /* make sure we can write to all the allocated memory */
+    memset(p, 'A', 8 * 14);
+
+    /* create another block after the first */
+    p2 = XtMalloc(73);
+    g_assert_nonnull(p2);
+
+    /* now resize the first */
+    p3 = XtReallocArray(p, 73, 14);
+    g_assert_nonnull(p3);
+    CHECK_ALIGNMENT(p3);
+    CHECK_SIZE(p3, 73 * 14);
+    /* verify previous values are still present */
+    for (int i = 0; i < (8 * 14); i++) {
+        g_assert_cmpint(p3[i], ==, 'A');
+    }
+
+    XtFree(p3);
+    XtFree(p2);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure XtReallocArray(0) returns a valid pointer as expected */
+static void test_XtReallocArray_zero(void)
+{
+    void *p, *p2;
+
+    errno = 0;
+
+    p = XtReallocArray(NULL, 0, 0);
+    g_assert_nonnull(p);
+
+    p2 = XtReallocArray(p, 0, 0);
+#ifdef MALLOC_0_RETURNS_NULL
+    g_assert_null(p);
+#else
+    g_assert_nonnull(p);
+#endif
+
+    XtFree(p2);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure sizes larger than the limit we set in main() fail */
+static void test_XtReallocArray_oversize(void)
+{
+    void *p, *p2;
+
+    /* Pick a number of elements between 1 & 16K */
+    guint32 num = g_test_rand_int_range(1, (16 * 1024));
+    /* Pick a size between 1 & 16K */
+    guint32 size = g_test_rand_int_range(1, (16 * 1024));
+
+    p = XtReallocArray(NULL, num, size);
+    g_assert_nonnull(p);
+    CHECK_ALIGNMENT(p);
+    CHECK_SIZE(p, num * size);
+
+    if (setjmp(jmp_env) == 0) {
+        p2 = XtReallocArray(p, 2, ALLOC_LIMIT);
+        g_assert_null(p2);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    errno = 0;
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure XtReallocArray catches integer overflow if possible, by requesting
+ * sizes that are so large that they cause overflows when either adding the
+ * reallocarray data block overhead or aligning.
+ *
+ * Testing integer overflow cases is limited by the fact that XtReallocArray
+ * only takes unsigned arguments (typically 32-bit), and relies on
+ * the underlying libc reallocarray to catch overflow, which can't occur if
+ * 32-bit arguments are passed to a function taking 64-bit args.
+ */
+static void test_XtReallocArray_overflow(void)
+{
+#if UINT_MAX < SIZE_MAX
+    g_test_skip("overflow not possible in this config");
+#else
+    void *p, *p2;
+
+    /* Pick a number of elements between 1 & 16K */
+    guint32 num = g_test_rand_int_range(1, (16 * 1024));
+    /* Pick a size between 1 & 16K */
+    guint32 size = g_test_rand_int_range(1, (16 * 1024));
+
+    p = XtReallocArray(NULL, num, size);
+    g_assert_nonnull(p);
+    CHECK_ALIGNMENT(p);
+    CHECK_SIZE(p, num * size);
+
+    if (setjmp(jmp_env) == 0) {
+        p2 = XtReallocArray(p, 1, SIZE_MAX);
+        g_assert_null(p2);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    if (setjmp(jmp_env) == 0) {
+        /* SQRT_SIZE_MAX * SQRT_SIZE_MAX == 0 due to overflow */
+        p2 = XtReallocArray(p, SQRT_SIZE_MAX, SQRT_SIZE_MAX);
+        g_assert_null(p2);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    if (setjmp(jmp_env) == 0) {
+        /* Overflows to a small positive number */
+        p2 = XtReallocArray(p, SQRT_SIZE_MAX + 1, SQRT_SIZE_MAX);
+        g_assert_null(p2);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    errno = 0;
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+#endif
+}
+
+
 int main(int argc, char** argv)
 {
     struct rlimit lim;
@@ -606,5 +766,12 @@ int main(int argc, char** argv)
     g_test_add_func("/Alloc/XtRealloc/oversize", test_XtRealloc_oversize);
     g_test_add_func("/Alloc/XtRealloc/overflow", test_XtRealloc_overflow);
 
+    g_test_add_func("/Alloc/XtReallocArray/normal", test_XtReallocArray_normal);
+    g_test_add_func("/Alloc/XtReallocArray/zero", test_XtReallocArray_zero);
+    g_test_add_func("/Alloc/XtReallocArray/oversize",
+                    test_XtReallocArray_oversize);
+    g_test_add_func("/Alloc/XtReallocArray/overflow",
+                    test_XtReallocArray_overflow);
+
     return g_test_run();
 }
diff --git a/test/Makefile.am b/test/Makefile.am
index adc7060..00229fb 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -5,7 +5,8 @@ check_PROGRAMS = Alloc Converters Event
 TESTS=$(check_PROGRAMS)
 
 AM_CFLAGS = $(CWARNFLAGS) $(XT_CFLAGS) $(GLIB_CFLAGS)
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include
+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include \
+    -I$(top_builddir)/include/X11
 LDADD= $(top_builddir)/src/libXt.la $(GLIB_LIBS)
 
 TESTS_ENVIRONMENT = $(MALLOC_DEBUG_ENV)