fix possible OOB or OOM in gdImageScale, reported by Secunia (CVE 2016-6207)

1 files changed, 40 insertions, 7 deletions

diff --git a/src/gd_interpolation.c b/src/gd_interpolation.c
index da6c8ad..72845d2 100644
--- a/src/gd_interpolation.c
+++ b/src/gd_interpolation.c
@@ -829,6 +829,7 @@ static inline LineContribType * _gdContributionsAlloc(unsigned int line_length,
 {
 	unsigned int u = 0;
 	LineContribType *res;
+	int overflow_error = 0;
 
 	res = (LineContribType *) gdMalloc(sizeof(LineContribType));
 	if (!res) {
@@ -836,10 +837,28 @@ static inline LineContribType * _gdContributionsAlloc(unsigned int line_length,
 	}
 	res->WindowSize = windows_size;
 	res->LineLength = line_length;
+	if (overflow2(line_length, sizeof(ContributionType))) {
+		return NULL;
+	}
 	res->ContribRow = (ContributionType *) gdMalloc(line_length * sizeof(ContributionType));
-
+	if (res->ContribRow == NULL) {
+		gdFree(res);
+		return NULL;
+	}
 	for (u = 0 ; u < line_length ; u++) {
-		res->ContribRow[u].Weights = (double *) gdMalloc(windows_size * sizeof(double));
+		if (overflow2(windows_size, sizeof(double))) {
+			overflow_error = 1;
+		} else {
+			res->ContribRow[u].Weights = (double *) gdMalloc(windows_size * sizeof(double));
+		}
+		if (overflow_error == 1 || res->ContribRow[u].Weights == NULL) {
+			u--;
+			while (u >= 0) {
+				gdFree(res->ContribRow[u].Weights);
+				u--;
+			}
+			return NULL;
+		}
 	}
 	return res;
 }
@@ -872,7 +891,9 @@ static inline LineContribType *_gdContributionsCalc(unsigned int line_size, unsi
 
 	windows_size = 2 * (int)ceil(width_d) + 1;
 	res = _gdContributionsAlloc(line_size, windows_size);
-
+	if (res == NULL) {
+		return NULL;
+	}
 	for (u = 0; u < line_size; u++) {
 		const double dCenter = (double)u / scale_d;
 		/* get the significant edge points affecting the pixel */
@@ -977,7 +998,6 @@ _gdScalePass(const gdImagePtr pSrc, const unsigned int src_len,
         _gdScaleOneAxis(pSrc, pDst, dst_len, line_ndx, contrib, axis);
 	}
 	_gdContributionsFree (contrib);
-
     return 1;
 }/* _gdScalePass*/
 
@@ -990,6 +1010,7 @@ gdImageScaleTwoPass(const gdImagePtr src, const unsigned int new_width,
     const unsigned int src_height = src->sy;
 	gdImagePtr tmp_im = NULL;
 	gdImagePtr dst = NULL;
+	int scale_pass_res;
 
     /* First, handle the trivial case. */
     if (src_width == new_width && src_height == new_height) {
@@ -1011,7 +1032,11 @@ gdImageScaleTwoPass(const gdImagePtr src, const unsigned int new_width,
         }
         gdImageSetInterpolationMethod(tmp_im, src->interpolation_id);
 
-        _gdScalePass(src, src_width, tmp_im, new_width, src_height, HORIZONTAL);
+		scale_pass_res = _gdScalePass(src, src_width, tmp_im, new_width, src_height, HORIZONTAL);
+		if (scale_pass_res != 1) {
+			gdImageDestroy(tmp_im);
+			return NULL;
+		}
     }/* if .. else*/
 
     /* If vertical sizes match, we're done. */
@@ -1024,10 +1049,18 @@ gdImageScaleTwoPass(const gdImagePtr src, const unsigned int new_width,
 	dst = gdImageCreateTrueColor(new_width, new_height);
 	if (dst != NULL) {
         gdImageSetInterpolationMethod(dst, src->interpolation_id);
-        _gdScalePass(tmp_im, src_height, dst, new_height, new_width, VERTICAL);
+        scale_pass_res = _gdScalePass(tmp_im, src_height, dst, new_height, new_width, VERTICAL);
+		if (scale_pass_res != 1) {
+			gdImageDestroy(dst);
+			if (src != tmp_im && tmp_im != NULL) {
+				gdImageDestroy(tmp_im);
+			}
+			return NULL;
+	   }
     }/* if */
 
-    if (src != tmp_im) {
+
+	if (src != tmp_im && tmp_im != NULL) {
         gdImageDestroy(tmp_im);
     }/* if */