allocations: test for overflow of requested size

Introduce some helper macros to test integer overflow from arithmetic
and set error message appropriately.

1 files changed, 20 insertions, 7 deletions

diff --git a/src/array.h b/src/array.h
index af9eafa43..ca5a35ffe 100644
--- a/src/array.h
+++ b/src/array.h
@@ -45,15 +45,28 @@ typedef git_array_t(char) git_array_generic_t;
 GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size)
 {
 	volatile git_array_generic_t *a = _a;
-	uint32_t new_size = (a->size < 8) ? 8 : a->asize * 3 / 2;
-	char *new_array = git__realloc(a->ptr, new_size * item_size);
-	if (!new_array) {
-		git_array_clear(*a);
-		return NULL;
+	uint32_t new_size;
+	char *new_array;
+
+	if (a->size < 8) {
+		new_size = 8;
 	} else {
-		a->ptr = new_array; a->asize = new_size; a->size++;
-		return a->ptr + (a->size - 1) * item_size;
+		if (GIT_ALLOC_OVERFLOW_MULTIPLY(a->size, 3 / 2))
+			goto on_oom;
+
+		new_size = a->size * 3 / 2;
 	}
+
+	if (GIT_ALLOC_OVERFLOW_MULTIPLY(new_size, item_size) ||
+		(new_array = git__realloc(a->ptr, new_size * item_size)) == NULL)
+		goto on_oom;
+
+	a->ptr = new_array; a->asize = new_size; a->size++;
+	return a->ptr + (a->size - 1) * item_size;
+
+on_oom:
+	git_array_clear(*a);
+	return NULL;
 }
 
 #define git_array_alloc(a) \