Fix size value wrap around in AO_malloc_large

AO_malloc(SIZE_MAX) should return NULL now.

* src/atomic_ops_malloc.c (SIZET_SAT_ADD): New macro.
* src/atomic_ops_malloc.c (AO_malloc_large): Use SIZET_SAT_ADD to
avoid integer overflow when computing the memory size to map (i.e.,
malloc should handle arguments close to SIZE_MAX correctly).

1 files changed, 6 insertions, 3 deletions

diff --git a/src/atomic_ops_malloc.c b/src/atomic_ops_malloc.c
index 124c87c..3753016 100644
--- a/src/atomic_ops_malloc.c
+++ b/src/atomic_ops_malloc.c
@@ -135,6 +135,10 @@ static char *get_mmaped(size_t sz)
   return result;
 }
 
+/* Saturated addition of size_t values.  Used to avoid value wrap       */
+/* around on overflow.  The arguments should have no side effects.      */
+#define SIZET_SAT_ADD(a, b) ((a) < ~(size_t)(b) ? (a) + (b) : ~(size_t)0)
+
 /* Allocate an object of size (incl. header) of size > CHUNK_SIZE.      */
 /* sz includes space for an AO_t-sized header.                          */
 static char *
@@ -142,9 +146,8 @@ AO_malloc_large(size_t sz)
 {
  char * result;
  /* The header will force us to waste ALIGNMENT bytes, incl. header.    */
-   sz += ALIGNMENT;
- /* Round to multiple of CHUNK_SIZE.    */
-   sz = (sz + CHUNK_SIZE - 1) & ~(CHUNK_SIZE - 1);
+ /* Round to multiple of CHUNK_SIZE.                                    */
+ sz = SIZET_SAT_ADD(sz, ALIGNMENT + CHUNK_SIZE - 1) & ~(CHUNK_SIZE - 1);
  result = get_mmaped(sz);
  if (result == 0) return 0;
  result += ALIGNMENT;