xdiff: convert size variables to size_t

3 files changed, 110 insertions, 50 deletions

diff --git a/src/xdiff/xdiffi.c b/src/xdiff/xdiffi.c
index 0620e5fff..f4d01b48c 100644
--- a/src/xdiff/xdiffi.c
+++ b/src/xdiff/xdiffi.c
@@ -21,7 +21,8 @@
  */
 
 #include "xinclude.h"
-
+#include "common.h"
+#include "integer.h"
 
 
 #define XDL_MAX_COST_MIN 256
@@ -323,7 +324,7 @@ int xdl_recs_cmp(diffdata_t *dd1, long off1, long lim1,
 
 int xdl_do_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
 		xdfenv_t *xe) {
-	long ndiags;
+	size_t ndiags, allocsize;
 	long *kvd, *kvdf, *kvdb;
 	xdalgoenv_t xenv;
 	diffdata_t dd1, dd2;
@@ -343,9 +344,12 @@ int xdl_do_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
 	 * Allocate and setup K vectors to be used by the differential algorithm.
 	 * One is to store the forward path and one to store the backward path.
 	 */
-	ndiags = xe->xdf1.nreff + xe->xdf2.nreff + 3;
-	if (!(kvd = (long *) xdl_malloc((2 * ndiags + 2) * sizeof(long)))) {
+	GITERR_CHECK_ALLOC_ADD3(&ndiags, xe->xdf1.nreff, xe->xdf2.nreff, 3);
+	GITERR_CHECK_ALLOC_MULTIPLY(&allocsize, ndiags, 2);
+	GITERR_CHECK_ALLOC_ADD(&allocsize, allocsize, 2);
+	GITERR_CHECK_ALLOC_MULTIPLY(&allocsize, allocsize, sizeof(long));
 
+	if (!(kvd = (long *) xdl_malloc(allocsize))) {
 		xdl_free_env(xe);
 		return -1;
 	}
diff --git a/src/xdiff/xhistogram.c b/src/xdiff/xhistogram.c
index 500d8112a..0c2edb89c 100644
--- a/src/xdiff/xhistogram.c
+++ b/src/xdiff/xhistogram.c
@@ -44,6 +44,7 @@
 #include "xinclude.h"
 #include "xtypes.h"
 #include "xdiff.h"
+#include "common.h"
 
 #define MAX_PTR	UINT_MAX
 #define MAX_CNT	UINT_MAX
@@ -271,7 +272,7 @@ static int histogram_diff(
 {
 	struct histindex index;
 	struct region lcs;
-	unsigned int sz;
+	size_t sz;
 	int result = -1;
 
 	if (count1 <= 0 && count2 <= 0)
@@ -302,7 +303,8 @@ static int histogram_diff(
 
 	index.table_bits = xdl_hashbits(count1);
 	sz = index.records_size = 1 << index.table_bits;
-	sz *= sizeof(struct record *);
+	GITERR_CHECK_ALLOC_MULTIPLY(&sz, sz, sizeof(struct record *));
+
 	if (!(index.records = (struct record **) xdl_malloc(sz)))
 		goto cleanup;
 	memset(index.records, 0, sz);
diff --git a/src/xdiff/xmerge.c b/src/xdiff/xmerge.c
index b11e59817..7b7e0e2d3 100644
--- a/src/xdiff/xmerge.c
+++ b/src/xdiff/xmerge.c
@@ -21,6 +21,7 @@
  */
 
 #include "xinclude.h"
+#include "common.h"
 
 typedef struct s_xdmerge {
 	struct s_xdmerge *next;
@@ -109,59 +110,74 @@ static int xdl_merge_cmp_lines(xdfenv_t *xe1, int i1, xdfenv_t *xe2, int i2,
 	return 0;
 }
 
-static int xdl_recs_copy_0(int use_orig, xdfenv_t *xe, int i, int count, int add_nl, char *dest)
+static int xdl_recs_copy_0(size_t *out, int use_orig, xdfenv_t *xe, int i, int count, int add_nl, char *dest)
 {
 	xrecord_t **recs;
-	int size = 0;
+	size_t size = 0;
+
+	*out = 0;
 
 	recs = (use_orig ? xe->xdf1.recs : xe->xdf2.recs) + i;
 
 	if (count < 1)
 		return 0;
 
-	for (i = 0; i < count; size += recs[i++]->size)
+	for (i = 0; i < count; ) {
 		if (dest)
 			memcpy(dest + size, recs[i]->ptr, recs[i]->size);
+
+		GITERR_CHECK_ALLOC_ADD(&size, size, recs[i++]->size);
+	}
+
 	if (add_nl) {
 		i = recs[count - 1]->size;
 		if (i == 0 || recs[count - 1]->ptr[i - 1] != '\n') {
 			if (dest)
 				dest[size] = '\n';
-			size++;
+
+			GITERR_CHECK_ALLOC_ADD(&size, size, 1);
 		}
 	}
-	return size;
+
+	*out = size;
+	return 0;
 }
 
-static int xdl_recs_copy(xdfenv_t *xe, int i, int count, int add_nl, char *dest)
+static int xdl_recs_copy(size_t *out, xdfenv_t *xe, int i, int count, int add_nl, char *dest)
 {
-	return xdl_recs_copy_0(0, xe, i, count, add_nl, dest);
+	return xdl_recs_copy_0(out, 0, xe, i, count, add_nl, dest);
 }
 
-static int xdl_orig_copy(xdfenv_t *xe, int i, int count, int add_nl, char *dest)
+static int xdl_orig_copy(size_t *out, xdfenv_t *xe, int i, int count, int add_nl, char *dest)
 {
-	return xdl_recs_copy_0(1, xe, i, count, add_nl, dest);
+	return xdl_recs_copy_0(out, 1, xe, i, count, add_nl, dest);
 }
 
-static int fill_conflict_hunk(xdfenv_t *xe1, const char *name1,
+static int fill_conflict_hunk(size_t *out, xdfenv_t *xe1, const char *name1,
 			      xdfenv_t *xe2, const char *name2,
 			      const char *name3,
-			      int size, int i, int style,
+			      size_t size, int i, int style,
 			      xdmerge_t *m, char *dest, int marker_size)
 {
 	int marker1_size = (name1 ? (int)strlen(name1) + 1 : 0);
 	int marker2_size = (name2 ? (int)strlen(name2) + 1 : 0);
 	int marker3_size = (name3 ? (int)strlen(name3) + 1 : 0);
+	size_t copied;
+
+	*out = 0;
 
 	if (marker_size <= 0)
 		marker_size = DEFAULT_CONFLICT_MARKER_SIZE;
 
 	/* Before conflicting part */
-	size += xdl_recs_copy(xe1, i, m->i1 - i, 0,
-			      dest ? dest + size : NULL);
+	if (xdl_recs_copy(&copied, xe1, i, m->i1 - i, 0,
+			      dest ? dest + size : NULL) < 0)
+		return -1;
+
+	GITERR_CHECK_ALLOC_ADD(&size, size, copied);
 
 	if (!dest) {
-		size += marker_size + 1 + marker1_size;
+		GITERR_CHECK_ALLOC_ADD4(&size, size, marker_size, 1, marker1_size);
 	} else {
 		memset(dest + size, '<', marker_size);
 		size += marker_size;
@@ -174,13 +190,16 @@ static int fill_conflict_hunk(xdfenv_t *xe1, const char *name1,
 	}
 
 	/* Postimage from side #1 */
-	size += xdl_recs_copy(xe1, m->i1, m->chg1, 1,
-			      dest ? dest + size : NULL);
+	if (xdl_recs_copy(&copied, xe1, m->i1, m->chg1, 1,
+			      dest ? dest + size : NULL) < 0)
+		return -1;
+
+	GITERR_CHECK_ALLOC_ADD(&size, size, copied);
 
 	if (style == XDL_MERGE_DIFF3) {
 		/* Shared preimage */
 		if (!dest) {
-			size += marker_size + 1 + marker3_size;
+			GITERR_CHECK_ALLOC_ADD4(&size, size, marker_size, 1, marker3_size);
 		} else {
 			memset(dest + size, '|', marker_size);
 			size += marker_size;
@@ -191,12 +210,15 @@ static int fill_conflict_hunk(xdfenv_t *xe1, const char *name1,
 			}
 			dest[size++] = '\n';
 		}
-		size += xdl_orig_copy(xe1, m->i0, m->chg0, 1,
-				      dest ? dest + size : NULL);
+
+		if (xdl_orig_copy(&copied, xe1, m->i0, m->chg0, 1,
+				      dest ? dest + size : NULL) < 0)
+			return -1;
+		GITERR_CHECK_ALLOC_ADD(&size, size, copied);
 	}
 
 	if (!dest) {
-		size += marker_size + 1;
+		GITERR_CHECK_ALLOC_ADD3(&size, size, marker_size, 1);
 	} else {
 		memset(dest + size, '=', marker_size);
 		size += marker_size;
@@ -204,10 +226,14 @@ static int fill_conflict_hunk(xdfenv_t *xe1, const char *name1,
 	}
 
 	/* Postimage from side #2 */
-	size += xdl_recs_copy(xe2, m->i2, m->chg2, 1,
-			      dest ? dest + size : NULL);
+
+	if (xdl_recs_copy(&copied, xe2, m->i2, m->chg2, 1,
+			      dest ? dest + size : NULL) < 0)
+		return -1;
+	GITERR_CHECK_ALLOC_ADD(&size, size, copied);
+
 	if (!dest) {
-		size += marker_size + 1 + marker2_size;
+		GITERR_CHECK_ALLOC_ADD4(&size, size, marker_size, 1, marker2_size);
 	} else {
 		memset(dest + size, '>', marker_size);
 		size += marker_size;
@@ -218,46 +244,69 @@ static int fill_conflict_hunk(xdfenv_t *xe1, const char *name1,
 		}
 		dest[size++] = '\n';
 	}
-	return size;
+
+	*out = size;
+	return 0;
 }
 
-static int xdl_fill_merge_buffer(xdfenv_t *xe1, const char *name1,
+static int xdl_fill_merge_buffer(size_t *out,
+				 xdfenv_t *xe1, const char *name1,
 				 xdfenv_t *xe2, const char *name2,
 				 const char *ancestor_name,
 				 int favor,
 				 xdmerge_t *m, char *dest, int style,
 				 int marker_size)
 {
-	int size, i;
+	size_t size, copied;
+	int i;
+
+	*out = 0;
 
 	for (size = i = 0; m; m = m->next) {
 		if (favor && !m->mode)
 			m->mode = favor;
 
-		if (m->mode == 0)
-			size = fill_conflict_hunk(xe1, name1, xe2, name2,
+		if (m->mode == 0) {
+			if (fill_conflict_hunk(&size, xe1, name1, xe2, name2,
 						  ancestor_name,
 						  size, i, style, m, dest,
-						  marker_size);
+						  marker_size) < 0)
+				return -1;
+		}
 		else if (m->mode & 3) {
 			/* Before conflicting part */
-			size += xdl_recs_copy(xe1, i, m->i1 - i, 0,
-					      dest ? dest + size : NULL);
+			if (xdl_recs_copy(&copied, xe1, i, m->i1 - i, 0,
+					      dest ? dest + size : NULL) < 0)
+				return -1;
+			GITERR_CHECK_ALLOC_ADD(&size, size, copied);
+
 			/* Postimage from side #1 */
-			if (m->mode & 1)
-				size += xdl_recs_copy(xe1, m->i1, m->chg1, (m->mode & 2),
-						      dest ? dest + size : NULL);
+			if (m->mode & 1) {
+				if (xdl_recs_copy(&copied, xe1, m->i1, m->chg1, (m->mode & 2),
+						      dest ? dest + size : NULL) < 0)
+					return -1;
+				GITERR_CHECK_ALLOC_ADD(&size, size, copied);
+			}
+
 			/* Postimage from side #2 */
-			if (m->mode & 2)
-				size += xdl_recs_copy(xe2, m->i2, m->chg2, 0,
-						      dest ? dest + size : NULL);
+			if (m->mode & 2) {
+				if (xdl_recs_copy(&copied, xe2, m->i2, m->chg2, 0,
+						      dest ? dest + size : NULL) < 0)
+					return -1;
+				GITERR_CHECK_ALLOC_ADD(&size, size, copied);
+			}
 		} else
 			continue;
 		i = m->i1 + m->chg1;
 	}
-	size += xdl_recs_copy(xe1, i, xe1->xdf2.nrec - i, 0,
-			      dest ? dest + size : NULL);
-	return size;
+
+	if (xdl_recs_copy(&copied, xe1, i, xe1->xdf2.nrec - i, 0,
+			      dest ? dest + size : NULL) < 0)
+		return -1;
+	GITERR_CHECK_ALLOC_ADD(&size, size, copied);
+
+	*out = size;
+	return 0;
 }
 
 /*
@@ -551,19 +600,24 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1,
 	/* output */
 	if (result) {
 		int marker_size = xmp->marker_size;
-		int size = xdl_fill_merge_buffer(xe1, name1, xe2, name2,
+		size_t size;
+
+		if (xdl_fill_merge_buffer(&size, xe1, name1, xe2, name2,
 						 ancestor_name,
 						 favor, changes, NULL, style,
-						 marker_size);
+						 marker_size) < 0)
+			return -1;
+
 		result->ptr = xdl_malloc(size);
 		if (!result->ptr) {
 			xdl_cleanup_merge(changes);
 			return -1;
 		}
 		result->size = size;
-		xdl_fill_merge_buffer(xe1, name1, xe2, name2,
+		if (xdl_fill_merge_buffer(&size, xe1, name1, xe2, name2,
 				      ancestor_name, favor, changes,
-				      result->ptr, style, marker_size);
+				      result->ptr, style, marker_size) < 0)
+			return -1;
 	}
 	return xdl_cleanup_merge(changes);
 }