Fix size value wrap around in AO_malloc_large

(Cherry-pick commits 1033478 and b1365bd from 'master' branch.)

AO_malloc(SIZE_MAX) should return NULL now.

* src/atomic_ops_malloc.c: Include limits.h (unless SIZE_MAX already
defined).
* src/atomic_ops_malloc.c (AO_SIZE_MAX): New macro.
* src/atomic_ops_malloc.c (SIZET_SAT_ADD): New macro.
* src/atomic_ops_malloc.c (AO_malloc_large): Use SIZET_SAT_ADD to
avoid integer overflow when computing the memory size to map (i.e.,
malloc should handle arguments close to SIZE_MAX correctly).

1 files changed, 16 insertions, 3 deletions

diff --git a/src/atomic_ops_malloc.c b/src/atomic_ops_malloc.c
index c2401f3..f8c18ec 100644
--- a/src/atomic_ops_malloc.c
+++ b/src/atomic_ops_malloc.c
@@ -130,6 +130,20 @@ static char *get_mmaped(size_t sz)
   return result;
 }
 
+#ifndef SIZE_MAX
+# include <limits.h>
+#endif
+#ifdef SIZE_MAX
+# define AO_SIZE_MAX SIZE_MAX
+#else
+# define AO_SIZE_MAX (~(size_t)0)
+#endif
+
+/* Saturated addition of size_t values.  Used to avoid value wrap       */
+/* around on overflow.  The arguments should have no side effects.      */
+#define SIZET_SAT_ADD(a, b) \
+                ((a) < AO_SIZE_MAX - (b) ? (a) + (b) : AO_SIZE_MAX)
+
 /* Allocate an object of size (incl. header) of size > CHUNK_SIZE.      */
 /* sz includes space for an AO_t-sized header.                          */
 static char *
@@ -137,9 +151,8 @@ AO_malloc_large(size_t sz)
 {
  char * result;
  /* The header will force us to waste ALIGNMENT bytes, incl. header.    */
-   sz += ALIGNMENT;
- /* Round to multiple of CHUNK_SIZE.    */
-   sz = (sz + CHUNK_SIZE - 1) & ~(CHUNK_SIZE - 1);
+ /* Round to multiple of CHUNK_SIZE.                                    */
+ sz = SIZET_SAT_ADD(sz, ALIGNMENT + CHUNK_SIZE - 1) & ~(CHUNK_SIZE - 1);
  result = get_mmaped(sz);
  if (result == 0) return 0;
  result += ALIGNMENT;