test: Add unit tests for XtMalloc, XtCalloc, & XtRealloc

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

3 files changed, 508 insertions, 1 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 325686d..0e09011 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -100,3 +100,8 @@ build:
     - make check
     - make distcheck
     - popd > /dev/null
+  artifacts:
+    when: always
+    reports:
+    paths:
+      - _builddir/test/*.log
diff --git a/configure.ac b/configure.ac
index 005ee70..e0dbaa4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -140,6 +140,10 @@ fi
 XORG_ENABLE_UNIT_TESTS
 XORG_WITH_GLIB([2.40])
 XORG_MEMORY_CHECK_FLAGS
+if test "x$enable_unit_tests" != "xno" ; then
+	AC_CHECK_FUNCS([malloc_usable_size])
+	AC_CHECK_HEADERS([malloc.h])
+fi
 
 # Replaces XFileSearchPathDefault from Imake configs
 XFILESEARCHPATHDEFAULT='$(sysconfdir)/X11/%L/%T/%N%C%S:$(sysconfdir)/X11/%l/%T/%N%C%S:$(sysconfdir)/X11/%T/%N%C%S:$(sysconfdir)/X11/%L/%T/%N%S:$(sysconfdir)/X11/%l/%T/%N%S:$(sysconfdir)/X11/%T/%N%S:$(datadir)/X11/%L/%T/%N%C%S:$(datadir)/X11/%l/%T/%N%C%S:$(datadir)/X11/%T/%N%C%S:$(datadir)/X11/%L/%T/%N%S:$(datadir)/X11/%l/%T/%N%S:$(datadir)/X11/%T/%N%S:$(libdir)/X11/%L/%T/%N%C%S:$(libdir)/X11/%l/%T/%N%C%S:$(libdir)/X11/%T/%N%C%S:$(libdir)/X11/%L/%T/%N%S:$(libdir)/X11/%l/%T/%N%S:$(libdir)/X11/%T/%N%S'
diff --git a/test/Alloc.c b/test/Alloc.c
index 591a003..3120f0f 100644
--- a/test/Alloc.c
+++ b/test/Alloc.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2022, Oracle and/or its affiliates.
+ * Copyright (c) 2011, 2023, Oracle and/or its affiliates.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -21,13 +21,56 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <X11/Xfuncproto.h>
 #include <X11/Intrinsic.h>
+#include <X11/IntrinsicI.h>
 #include <glib.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
+#include <errno.h>
+#include <limits.h>
+#include <setjmp.h>
+#include <sys/resource.h>
+#ifdef HAVE_MALLOC_H
+# include <malloc.h>
+#endif
 
 static const char *program_name;
 
+#ifndef g_assert_no_errno /* defined in glib 2.66 & later*/
+#define g_assert_no_errno(expr) g_assert_cmpint((expr), >=, 0)
+#endif
+
+/*
+ * Check that allocations point to properly aligned memory.
+ * For libXt, we consider that to be aligned to an 8-byte (64-bit) boundary.
+ */
+#define EXPECTED_ALIGNMENT 8
+
+#define CHECK_ALIGNMENT(ptr) \
+    g_assert_cmpint(((uintptr_t)ptr) % EXPECTED_ALIGNMENT, ==, 0)
+
+/* Check that allocations point to expected amounts of memory, as best we can. */
+#ifdef HAVE_MALLOC_USABLE_SIZE
+# define CHECK_SIZE(ptr, size) do {		\
+    size_t ps = malloc_usable_size(ptr);	\
+    g_assert_cmpint(ps, >=, (size));		\
+} while (0)
+#else
+# define CHECK_SIZE(ptr, size) *(((char *)ptr) + ((size) - 1)) = 0
+#endif
+
+/* Limit we set for memory allocation to be able to test failure cases */
+#define ALLOC_LIMIT (INT_MAX / 4)
+
+/* Square root of SIZE_MAX+1 */
+#define SQRT_SIZE_MAX ((size_t)1 << (sizeof (size_t) * 4))
+
 /* Just a long string of characters to pull from */
 const char test_chars[] =
     "|000 nul|001 soh|002 stx|003 etx|004 eot|005 enq|006 ack|007 bel|"
@@ -64,6 +107,22 @@ const char test_chars[] =
     "| 78  x | 79  y | 7a  z | 7b  { | 7c  | | 7d  } | 7e  ~ | 7f del|";
 
 
+/* Environment saved by setjmp() */
+static jmp_buf jmp_env;
+
+static void _X_NORETURN
+xt_error_handler(String message)
+{
+    if (message && *message)
+        fprintf(stderr, "Caught Error: %s\n", message);
+    else
+        fputs("Caught Error.\n", stderr);
+
+    /* Avoid exit() in XtErrorMsg() */
+    longjmp(jmp_env, 1);
+}
+
+
 /* Test a simple short string & int */
 static void test_XtAsprintf_short(void)
 {
@@ -98,15 +157,454 @@ static void test_XtAsprintf_long(void)
     g_assert_cmpint(asbuf[aslen], ==, '\0');
 }
 
+/* Make sure XtMalloc() works for a non-zero amount of memory */
+static void test_XtMalloc_normal(void)
+{
+    void *p;
+    /* Pick a size between 1 & 256K */
+    guint32 size = g_test_rand_int_range(1, (256 * 1024));
+
+    errno = 0;
+
+    p = XtMalloc(size);
+    g_assert_nonnull(p);
+    CHECK_ALIGNMENT(p);
+    CHECK_SIZE(p, size);
+
+    /* make sure we can write to all the allocated memory */
+    memset(p, 'A', size);
+
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure XtMalloc(0) returns expected results */
+static void test_XtMalloc_zero(void)
+{
+    void *p;
+
+    errno = 0;
+
+    p = XtMalloc(0);
+#if !defined(MALLOC_0_RETURNS_NULL) || defined(XTMALLOC_BC)
+    g_assert_nonnull(p);
+#else
+    g_assert_null(p);
+#endif
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+
+    /* __XtMalloc always returns a non-NULL pointer for size == 0 */
+    p = __XtMalloc(0);
+    g_assert_nonnull(p);
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure sizes larger than the limit we set in main() fail */
+static void test_XtMalloc_oversize(void)
+{
+    void *p;
+
+    if (setjmp(jmp_env) == 0) {
+        p = XtMalloc(UINT_MAX - 1);
+        g_assert_null(p);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+}
+
+/* Make sure XtMalloc catches integer overflow if possible, by requesting
+ * sizes that are so large that they cause overflows when either adding the
+ * malloc data block overhead or aligning.
+ *
+ * Testing integer overflow cases is limited by the fact that XtMalloc
+ * only takes unsigned arguments (typically 32-bit), and relies on
+ * the underlying libc malloc to catch overflow, which can't occur if
+ * 32-bit arguments are passed to a function taking 64-bit args.
+ */
+static void test_XtMalloc_overflow(void)
+{
+#if UINT_MAX < SIZE_MAX
+    g_test_skip("overflow not possible in this config");
+#else
+    void *p;
+
+    if (setjmp(jmp_env) == 0) {
+        p = XtMalloc(SIZE_MAX);
+        g_assert_null(p);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    if (setjmp(jmp_env) == 0) {
+        p = XtMalloc(SIZE_MAX - 1);
+        g_assert_null(p);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    if (setjmp(jmp_env) == 0) {
+        p = XtMalloc(SIZE_MAX - 8);
+        g_assert_null(p);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+#endif
+}
+
+
+
+/* Make sure XtCalloc() works for a non-zero amount of memory */
+static void test_XtCalloc_normal(void)
+{
+    char *p;
+    /* Pick a number of elements between 1 & 16K */
+    guint32 num = g_test_rand_int_range(1, (16 * 1024));
+    /* Pick a size between 1 & 16K */
+    guint32 size = g_test_rand_int_range(1, (16 * 1024));
+
+    errno = 0;
+
+    p = XtCalloc(num, size);
+    g_assert_nonnull(p);
+    CHECK_ALIGNMENT(p);
+    CHECK_SIZE(p, num * size);
+
+    /* make sure all the memory was zeroed */
+    for (guint32 i = 0; i < (num * size); i++) {
+        g_assert_cmpint(p[i], ==, 0);
+    }
+
+    /* make sure we can write to all the allocated memory */
+    memset(p, 'A', num * size);
+
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure XtCalloc() returns the expected results for args of 0 */
+static void test_XtCalloc_zero(void)
+{
+    void *p;
+
+    errno = 0;
+
+    p = XtCalloc(0, 0);
+#if !defined(MALLOC_0_RETURNS_NULL) || defined(XTMALLOC_BC)
+    g_assert_nonnull(p);
+    XtFree(p);
+#else
+    g_assert_null(p);
+#endif
+    g_assert_cmpint(errno, ==, 0);
+
+    p = XtCalloc(1, 0);
+#if !defined(MALLOC_0_RETURNS_NULL) || defined(XTMALLOC_BC)
+    g_assert_nonnull(p);
+    XtFree(p);
+#else
+    g_assert_null(p);
+#endif
+    g_assert_cmpint(errno, ==, 0);
+
+    p = XtCalloc(0, 1);
+#if !defined(MALLOC_0_RETURNS_NULL)
+    g_assert_nonnull(p);
+    XtFree(p);
+#else
+    g_assert_null(p);
+#endif
+    g_assert_cmpint(errno, ==, 0);
+
+    /* __XtCalloc always returns a non-NULL pointer for size == 0 */
+    p = __XtCalloc(1, 0);
+    g_assert_nonnull(p);
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure sizes larger than the limit we set in main() fail. */
+static void test_XtCalloc_oversize(void)
+{
+    void *p;
+
+    if (setjmp(jmp_env) == 0) {
+        p = XtCalloc(2, ALLOC_LIMIT);
+        g_assert_null(p);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+}
+
+/* Make sure XtCalloc catches integer overflow if possible
+ *
+ * Testing integer overflow cases is limited by the fact that XtCalloc
+ * only takes unsigned arguments (typically 32-bit), and relies on
+ * the underlying libc calloc to catch overflow, which can't occur
+ * if 32-bit arguments are passed to a function taking 64-bit args.
+ */
+static void test_XtCalloc_overflow(void)
+{
+#if UINT_MAX < SIZE_MAX
+    g_test_skip("overflow not possible in this config");
+#else
+    void *p;
+
+    if (setjmp(jmp_env) == 0) {
+        p = XtCalloc(2, SIZE_MAX);
+        g_assert_null(p);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    if (setjmp(jmp_env) == 0) {
+        /* SQRT_SIZE_MAX * SQRT_SIZE_MAX == 0 due to overflow */
+        p = XtCalloc(SQRT_SIZE_MAX, SQRT_SIZE_MAX);
+        g_assert_null(p);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    if (setjmp(jmp_env) == 0) {
+        /* Overflows to a small positive number */
+        p = XtCalloc(SQRT_SIZE_MAX + 1, SQRT_SIZE_MAX);
+        g_assert_null(p);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+#endif
+}
+
+/* Make sure XtRealloc() works for a non-zero amount of memory */
+static void test_XtRealloc_normal(void)
+{
+    void *p, *p2;
+    char *p3;
+
+    errno = 0;
+
+    /* Make sure realloc with a NULL pointer acts as malloc */
+    p = XtRealloc(NULL, 814);
+    g_assert_nonnull(p);
+    CHECK_ALIGNMENT(p);
+    CHECK_SIZE(p, 814);
+
+    /* make sure we can write to all the allocated memory */
+    memset(p, 'A', 814);
+
+    /* create another block after the first */
+    p2 = XtMalloc(73);
+    g_assert_nonnull(p2);
+
+    /* now resize the first */
+    p3 = XtRealloc(p, 7314);
+    g_assert_nonnull(p3);
+    CHECK_ALIGNMENT(p3);
+    CHECK_SIZE(p3, 7314);
+
+    /* verify previous values are still present */
+    for (int i = 0; i < 814; i++) {
+        g_assert_cmpint(p3[i], ==, 'A');
+    }
+
+    XtFree(p3);
+    XtFree(p2);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure XtRealloc(0) returns a valid pointer as expected */
+static void test_XtRealloc_zero(void)
+{
+    void *p, *p2;
+
+    errno = 0;
+
+    p = XtRealloc(NULL, 0);
+    g_assert_nonnull(p);
+
+    p2 = XtRealloc(p, 0);
+#ifdef MALLOC_0_RETURNS_NULL
+    g_assert_null(p);
+#else
+    g_assert_nonnull(p);
+#endif
+
+    XtFree(p2);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure sizes larger than the limit we set in main() fail */
+static void test_XtRealloc_oversize(void)
+{
+    void *p, *p2;
+
+    /* Pick a size between 1 & 256K */
+    guint32 size = g_test_rand_int_range(1, (256 * 1024));
+
+    p = XtRealloc(NULL, size);
+    g_assert_nonnull(p);
+    CHECK_ALIGNMENT(p);
+
+    if (setjmp(jmp_env) == 0) {
+        p2 = XtRealloc(p, ALLOC_LIMIT + 1);
+        g_assert_null(p2);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    errno = 0;
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+}
+
+/* Make sure XtRealloc catches integer overflow if possible, by requesting
+ * sizes that are so large that they cause overflows when either adding the
+ * realloc data block overhead or aligning.
+ *
+ * Testing integer overflow cases is limited by the fact that XtRealloc
+ * only takes unsigned arguments (typically 32-bit), and relies on
+ * the underlying libc realloc to catch overflow, which can't occur if
+ * 32-bit arguments are passed to a function taking 64-bit args.
+ */
+static void test_XtRealloc_overflow(void)
+{
+#if UINT_MAX < SIZE_MAX
+    g_test_skip("overflow not possible in this config");
+#else
+    void *p, *p2;
+
+    /* Pick a size between 1 & 256K */
+    guint32 size = g_test_rand_int_range(1, (256 * 1024));
+
+    p = XtRealloc(NULL, size);
+    g_assert_nonnull(p);
+    CHECK_ALIGNMENT(p);
+
+    if (setjmp(jmp_env) == 0) {
+        p2 = XtRealloc(p, SIZE_MAX);
+        g_assert_null(p2);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    if (setjmp(jmp_env) == 0) {
+        p2 = XtRealloc(p, SIZE_MAX - 1);
+        g_assert_null(p2);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    if (setjmp(jmp_env) == 0) {
+        p2 = XtRealloc(p, SIZE_MAX - 8);
+        g_assert_null(p2);
+    } else {
+        /*
+         * We jumped here from error handler as expected.
+         * We cannot verify errno here, as the Xt error handler makes
+         * calls that override errno, when trying to load error message
+         * files from different locations.
+         */
+    }
+
+    errno = 0;
+    XtFree(p);
+    g_assert_cmpint(errno, ==, 0);
+#endif
+}
+
+
 int main(int argc, char** argv)
 {
+    struct rlimit lim;
+
     program_name = argv[0];
 
     g_test_init(&argc, &argv, NULL);
     g_test_bug_base("https://gitlab.freedesktop.org/xorg/lib/libxt/-/issues/");
 
+    /* Set a memory limit so we can test allocations over that size fail */
+    g_assert_no_errno(getrlimit(RLIMIT_AS, &lim));
+    if (lim.rlim_cur > ALLOC_LIMIT) {
+        lim.rlim_cur = ALLOC_LIMIT;
+        g_assert_no_errno(setrlimit(RLIMIT_AS, &lim));
+    }
+
+    /* Install xt_error_handler to avoid exit on allocation failure */
+    XtSetErrorHandler(xt_error_handler);
+
     g_test_add_func("/Alloc/XtAsprintf/short", test_XtAsprintf_short);
     g_test_add_func("/Alloc/XtAsprintf/long", test_XtAsprintf_long);
 
+    g_test_add_func("/Alloc/XtMalloc/normal", test_XtMalloc_normal);
+    g_test_add_func("/Alloc/XtMalloc/zero", test_XtMalloc_zero);
+    g_test_add_func("/Alloc/XtMalloc/oversize", test_XtMalloc_oversize);
+    g_test_add_func("/Alloc/XtMalloc/overflow", test_XtMalloc_overflow);
+
+    g_test_add_func("/Alloc/XtCalloc/normal", test_XtCalloc_normal);
+    g_test_add_func("/Alloc/XtCalloc/zero", test_XtCalloc_zero);
+    g_test_add_func("/Alloc/XtCalloc/oversize", test_XtCalloc_oversize);
+    g_test_add_func("/Alloc/XtCalloc/overflow", test_XtCalloc_overflow);
+
+    g_test_add_func("/Alloc/XtRealloc/normal", test_XtRealloc_normal);
+    g_test_add_func("/Alloc/XtRealloc/zero", test_XtRealloc_zero);
+    g_test_add_func("/Alloc/XtRealloc/oversize", test_XtRealloc_oversize);
+    g_test_add_func("/Alloc/XtRealloc/overflow", test_XtRealloc_overflow);
+
     return g_test_run();
 }