Add XtReallocArray() for overflow checking of multiplied args

Uses reallocarray() if available, otherwise checks for overflow itself,
if overflow is possible (i.e. in ILP32 & ILP64 environments, but not LP64
with 32-bit ints).

Includes unit tests and XtMallocArray() helper macro.

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

1 files changed, 51 insertions, 1 deletions

diff --git a/src/Alloc.c b/src/Alloc.c
index ce0594c..5bb27ce 100644
--- a/src/Alloc.c
+++ b/src/Alloc.c
@@ -1,5 +1,5 @@
 /***********************************************************
-Copyright (c) 1993, 2011, Oracle and/or its affiliates.
+Copyright (c) 1993, 2023, Oracle and/or its affiliates.
 
 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the "Software"),
@@ -84,6 +84,7 @@ in this Software without prior written authorization from The Open Group.
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>
+#include <limits.h>
 
 #define Xmalloc(size) malloc((size))
 #define Xrealloc(ptr, size) realloc((ptr), (size))
@@ -197,6 +198,38 @@ XtRealloc(char *ptr, unsigned size)
     return (ptr);
 }
 
+void *
+XtReallocArray(void *ptr, unsigned num, unsigned size)
+{
+    if (ptr == NULL) {
+#ifdef MALLOC_0_RETURNS_NULL
+        if ((num == 0) || (size == 0))
+            num = size = 1;
+#endif
+#if (SIZE_MAX / UINT_MAX) <= UINT_MAX
+        if ((num > 0) && (SIZE_MAX / num) < size)
+            _XtAllocError("reallocarray: overflow detected");
+#endif
+        return (XtMalloc(num * size));
+    }
+    else {
+        void *new;
+
+#ifdef HAVE_REALLOCARRAY
+        new = reallocarray(ptr, size, num);
+#else
+# if (SIZE_MAX / UINT_MAX) <= UINT_MAX
+        if ((num > 0) && ((SIZE_MAX / num) < size))
+            _XtAllocError("reallocarray: overflow detected");
+# endif
+        new = Xrealloc(ptr, size * num);
+#endif
+        if ((new == NULL) && (num != 0) && (size != 0))
+            _XtAllocError("reallocarray");
+        return (new);
+    }
+}
+
 char *
 XtCalloc(unsigned num, unsigned size)
 {
@@ -402,6 +435,23 @@ XtRealloc(char *ptr, unsigned size)
     return _XtRealloc(ptr, size, (char *) NULL, 0);
 }
 
+void *
+_XtReallocArray(void *ptr, unsigned num, unsigned size, const char *file, int line)
+{
+#if (SIZE_MAX / UINT_MAX) <= UINT_MAX
+    if ((num > 0) && (SIZE_MAX / num) < size)
+        _XtAllocError("reallocarray: overflow");
+#endif
+
+    return _XtRealloc(ptr, (num * size), file, line);
+}
+
+void *
+XtReallocArray(void *ptr, unsigned num, unsigned size)
+{
+    return _XtReallocArray(ptr, num, size, (char *) NULL, 0);
+}
+
 char *
 _XtCalloc(unsigned num, unsigned size, const char *file, int line)
 {