Back-patch recent security fixes for tif_pixarlog.c.

2 files changed, 62 insertions, 39 deletions

diff --git a/ChangeLog b/ChangeLog
index 33227e3e..6d6d2d19 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2012-12-10  Tom Lane  <tgl@sss.pgh.pa.us>
+
+	* libtiff/tif_pixarlog.c: Back-patch recent security fixes for
+	tif_pixarlog.c, namely the fix for CVE-2012-4447 and protections
+	against accessing outside the lookup arrays for out of range
+	inputs.
+
 2012-11-18  Bob Friesenhahn  <bfriesen@simple.dallas.tx.us>
 
 	* automake: Update Automake to 1.12.5 release.
diff --git a/libtiff/tif_pixarlog.c b/libtiff/tif_pixarlog.c
index ed8eb402..6e127fb6 100644
--- a/libtiff/tif_pixarlog.c
+++ b/libtiff/tif_pixarlog.c
@@ -1,4 +1,4 @@
-/* $Id: tif_pixarlog.c,v 1.15.2.4 2010-06-08 18:50:42 bfriesen Exp $ */
+/* $Id: tif_pixarlog.c,v 1.15.2.5 2012-12-10 17:38:54 tgl Exp $ */
 
 /*
  * Copyright (c) 1996-1997 Sam Leffler
@@ -117,9 +117,9 @@ horizontalAccumulateF(uint16 *wp, int n, int stride, float *op,
     if (n >= stride) {
 	mask = CODE_MASK;
 	if (stride == 3) {
-	    t0 = ToLinearF[cr = wp[0]];
-	    t1 = ToLinearF[cg = wp[1]];
-	    t2 = ToLinearF[cb = wp[2]];
+	    t0 = ToLinearF[cr = (wp[0] & mask)];
+	    t1 = ToLinearF[cg = (wp[1] & mask)];
+	    t2 = ToLinearF[cb = (wp[2] & mask)];
 	    op[0] = t0;
 	    op[1] = t1;
 	    op[2] = t2;
@@ -136,10 +136,10 @@ horizontalAccumulateF(uint16 *wp, int n, int stride, float *op,
 		op[2] = t2;
 	    }
 	} else if (stride == 4) {
-	    t0 = ToLinearF[cr = wp[0]];
-	    t1 = ToLinearF[cg = wp[1]];
-	    t2 = ToLinearF[cb = wp[2]];
-	    t3 = ToLinearF[ca = wp[3]];
+	    t0 = ToLinearF[cr = (wp[0] & mask)];
+	    t1 = ToLinearF[cg = (wp[1] & mask)];
+	    t2 = ToLinearF[cb = (wp[2] & mask)];
+	    t3 = ToLinearF[ca = (wp[3] & mask)];
 	    op[0] = t0;
 	    op[1] = t1;
 	    op[2] = t2;
@@ -183,9 +183,9 @@ horizontalAccumulate12(uint16 *wp, int n, int stride, int16 *op,
     if (n >= stride) {
 	mask = CODE_MASK;
 	if (stride == 3) {
-	    t0 = ToLinearF[cr = wp[0]] * SCALE12;
-	    t1 = ToLinearF[cg = wp[1]] * SCALE12;
-	    t2 = ToLinearF[cb = wp[2]] * SCALE12;
+	    t0 = ToLinearF[cr = (wp[0] & mask)] * SCALE12;
+	    t1 = ToLinearF[cg = (wp[1] & mask)] * SCALE12;
+	    t2 = ToLinearF[cb = (wp[2] & mask)] * SCALE12;
 	    op[0] = CLAMP12(t0);
 	    op[1] = CLAMP12(t1);
 	    op[2] = CLAMP12(t2);
@@ -202,10 +202,10 @@ horizontalAccumulate12(uint16 *wp, int n, int stride, int16 *op,
 		op[2] = CLAMP12(t2);
 	    }
 	} else if (stride == 4) {
-	    t0 = ToLinearF[cr = wp[0]] * SCALE12;
-	    t1 = ToLinearF[cg = wp[1]] * SCALE12;
-	    t2 = ToLinearF[cb = wp[2]] * SCALE12;
-	    t3 = ToLinearF[ca = wp[3]] * SCALE12;
+	    t0 = ToLinearF[cr = (wp[0] & mask)] * SCALE12;
+	    t1 = ToLinearF[cg = (wp[1] & mask)] * SCALE12;
+	    t2 = ToLinearF[cb = (wp[2] & mask)] * SCALE12;
+	    t3 = ToLinearF[ca = (wp[3] & mask)] * SCALE12;
 	    op[0] = CLAMP12(t0);
 	    op[1] = CLAMP12(t1);
 	    op[2] = CLAMP12(t2);
@@ -247,9 +247,9 @@ horizontalAccumulate16(uint16 *wp, int n, int stride, uint16 *op,
     if (n >= stride) {
 	mask = CODE_MASK;
 	if (stride == 3) {
-	    op[0] = ToLinear16[cr = wp[0]];
-	    op[1] = ToLinear16[cg = wp[1]];
-	    op[2] = ToLinear16[cb = wp[2]];
+	    op[0] = ToLinear16[cr = (wp[0] & mask)];
+	    op[1] = ToLinear16[cg = (wp[1] & mask)];
+	    op[2] = ToLinear16[cb = (wp[2] & mask)];
 	    n -= 3;
 	    while (n > 0) {
 		wp += 3;
@@ -260,10 +260,10 @@ horizontalAccumulate16(uint16 *wp, int n, int stride, uint16 *op,
 		op[2] = ToLinear16[(cb += wp[2]) & mask];
 	    }
 	} else if (stride == 4) {
-	    op[0] = ToLinear16[cr = wp[0]];
-	    op[1] = ToLinear16[cg = wp[1]];
-	    op[2] = ToLinear16[cb = wp[2]];
-	    op[3] = ToLinear16[ca = wp[3]];
+	    op[0] = ToLinear16[cr = (wp[0] & mask)];
+	    op[1] = ToLinear16[cg = (wp[1] & mask)];
+	    op[2] = ToLinear16[cb = (wp[2] & mask)];
+	    op[3] = ToLinear16[ca = (wp[3] & mask)];
 	    n -= 4;
 	    while (n > 0) {
 		wp += 4;
@@ -342,9 +342,9 @@ horizontalAccumulate8(uint16 *wp, int n, int stride, unsigned char *op,
     if (n >= stride) {
 	mask = CODE_MASK;
 	if (stride == 3) {
-	    op[0] = ToLinear8[cr = wp[0]];
-	    op[1] = ToLinear8[cg = wp[1]];
-	    op[2] = ToLinear8[cb = wp[2]];
+	    op[0] = ToLinear8[cr = (wp[0] & mask)];
+	    op[1] = ToLinear8[cg = (wp[1] & mask)];
+	    op[2] = ToLinear8[cb = (wp[2] & mask)];
 	    n -= 3;
 	    while (n > 0) {
 		n -= 3;
@@ -355,10 +355,10 @@ horizontalAccumulate8(uint16 *wp, int n, int stride, unsigned char *op,
 		op[2] = ToLinear8[(cb += wp[2]) & mask];
 	    }
 	} else if (stride == 4) {
-	    op[0] = ToLinear8[cr = wp[0]];
-	    op[1] = ToLinear8[cg = wp[1]];
-	    op[2] = ToLinear8[cb = wp[2]];
-	    op[3] = ToLinear8[ca = wp[3]];
+	    op[0] = ToLinear8[cr = (wp[0] & mask)];
+	    op[1] = ToLinear8[cg = (wp[1] & mask)];
+	    op[2] = ToLinear8[cb = (wp[2] & mask)];
+	    op[3] = ToLinear8[ca = (wp[3] & mask)];
 	    n -= 4;
 	    while (n > 0) {
 		n -= 4;
@@ -393,9 +393,9 @@ horizontalAccumulate8abgr(uint16 *wp, int n, int stride, unsigned char *op,
 	mask = CODE_MASK;
 	if (stride == 3) {
 	    op[0] = 0;
-	    t1 = ToLinear8[cb = wp[2]];
-	    t2 = ToLinear8[cg = wp[1]];
-	    t3 = ToLinear8[cr = wp[0]];
+	    t1 = ToLinear8[cb = (wp[2] & mask)];
+	    t2 = ToLinear8[cg = (wp[1] & mask)];
+	    t3 = ToLinear8[cr = (wp[0] & mask)];
 	    op[1] = t1;
 	    op[2] = t2;
 	    op[3] = t3;
@@ -413,10 +413,10 @@ horizontalAccumulate8abgr(uint16 *wp, int n, int stride, unsigned char *op,
 		op[3] = t3;
 	    }
 	} else if (stride == 4) {
-	    t0 = ToLinear8[ca = wp[3]];
-	    t1 = ToLinear8[cb = wp[2]];
-	    t2 = ToLinear8[cg = wp[1]];
-	    t3 = ToLinear8[cr = wp[0]];
+	    t0 = ToLinear8[ca = (wp[3] & mask)];
+	    t1 = ToLinear8[cb = (wp[2] & mask)];
+	    t2 = ToLinear8[cg = (wp[1] & mask)];
+	    t3 = ToLinear8[cr = (wp[0] & mask)];
 	    op[0] = t0;
 	    op[1] = t1;
 	    op[2] = t2;
@@ -630,10 +630,10 @@ PixarLogGuessDataFmt(TIFFDirectory *td)
 	return guess;
 }
 
-static uint32
-multiply(size_t m1, size_t m2)
+static tsize_t
+multiply(tsize_t m1, tsize_t m2)
 {
-	uint32	bytes = m1 * m2;
+	tsize_t	bytes = m1 * m2;
 
 	if (m1 && bytes / m1 != m2)
 		bytes = 0;
@@ -641,6 +641,20 @@ multiply(size_t m1, size_t m2)
 	return bytes;
 }
 
+static tsize_t
+add_ms(tsize_t m1, tsize_t m2)
+{
+	tsize_t bytes = m1 + m2;
+
+	/* if either input is zero, assume overflow already occurred */
+	if (m1 == 0 || m2 == 0)
+		bytes = 0;
+	else if (bytes <= m1 || bytes <= m2)
+		bytes = 0;
+
+	return bytes;
+}
+
 static int
 PixarLogSetupDecode(TIFF* tif)
 {
@@ -661,6 +675,8 @@ PixarLogSetupDecode(TIFF* tif)
 	    td->td_samplesperpixel : 1);
 	tbuf_size = multiply(multiply(multiply(sp->stride, td->td_imagewidth),
 				      td->td_rowsperstrip), sizeof(uint16));
+	/* add one more stride in case input ends mid-stride */
+	tbuf_size = add_ms(tbuf_size, sizeof(uint16) * sp->stride);
 	if (tbuf_size == 0)
 		return (0);
 	sp->tbuf = (uint16 *) _TIFFmalloc(tbuf_size);