BR 3392597: the system malloc() can return NULL

malloc(0) can legitimately return NULL; it does on some systems and
not others. Force the size to 1 byte if the size is 0 coming in,
except for realloc() where this is legitimate and equivalent to
free().

Since this is an abnormal case, and can't even happen with most C
libraries, handle it on the error path, after we already got back a
NULL pointer.

Reported-by: Ozkan Sezer <sezeroz@gmail.com>
Signed-off-by: H. Peter Anvin (Intel) <hpa@zytor.com>

1 files changed, 41 insertions, 6 deletions

diff --git a/nasmlib/alloc.c b/nasmlib/alloc.c
index 4e0ff9fe..b0d623a2 100644
--- a/nasmlib/alloc.c
+++ b/nasmlib/alloc.c
@@ -1,6 +1,6 @@
 /* ----------------------------------------------------------------------- *
  *
- *   Copyright 1996-2018 The NASM Authors - All Rights Reserved
+ *   Copyright 1996-2019 The NASM Authors - All Rights Reserved
  *   See the file AUTHORS included with the NASM distribution for
  *   the specific copyright holders.
  *
@@ -65,22 +65,57 @@ no_return nasm_alloc_failed(void)
 
 void *nasm_malloc(size_t size)
 {
-    return validate_ptr(malloc(size));
+    void *p;
+
+again:
+    p = malloc(size);
+
+    if (unlikely(!p)) {
+        if (!size) {
+            size = 1;
+            goto again;
+        }
+        nasm_alloc_failed();
+    }
+    return p;
 }
 
-void *nasm_calloc(size_t size, size_t nelem)
+void *nasm_calloc(size_t nelem, size_t size)
 {
-    return validate_ptr(calloc(size, nelem));
+    void *p;
+
+again:
+    p = calloc(nelem, size);
+
+    if (unlikely(!p)) {
+        if (!nelem || !size) {
+            nelem = size = 1;
+            goto again;
+        }
+        nasm_alloc_failed();
+    }
+
+    return p;
 }
 
 void *nasm_zalloc(size_t size)
 {
-    return validate_ptr(calloc(1, size));
+    return nasm_calloc(size, 1);
 }
 
+/*
+ * Unlike the system realloc, we do *not* allow size == 0 to be
+ * the equivalent to free(); we guarantee returning a non-NULL pointer.
+ *
+ * The check for calling malloc() is theoretically redundant, but be
+ * paranoid about the system library...
+ */
 void *nasm_realloc(void *q, size_t size)
 {
-    return validate_ptr(q ? realloc(q, size) : malloc(size));
+    if (unlikely(!size))
+        size = 1;
+    q = q ? realloc(q, size) : malloc(size);
+    return validate_ptr(q);
 }
 
 void nasm_free(void *q)