Fix CVE-2012-2088 and CVE-2012-2113

4 files changed, 163 insertions, 46 deletions

diff --git a/ChangeLog b/ChangeLog
index 63a8d0ac..de90dac3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2012-06-15  Tom Lane  <tgl@sss.pgh.pa.us>
+
+	* libtiff/tif_strip.c, libtiff/tif_tile.c: Back-patch the 4.0
+	behavior of treating signed overflow as an error in TIFFVStripSize
+	and TIFFVTileSize.  This is needed since the result is declared as
+	tsize_t which is signed, and callers are likely to do the wrong
+	thing entirely when the returned value is negative (CVE-2012-2088).
+
+	* tools/tiff2pdf.c: Defend against integer overflows while
+	calculating required buffer sizes (CVE-2012-2113).
+
 2012-06-04  Frank Warmerdam  <warmerdam@google.com>
 
 	* libtiff/tif_dirread.c: Avoid trusting samplesperpixel's default
diff --git a/libtiff/tif_strip.c b/libtiff/tif_strip.c
index 0542d004..1a50e1a0 100644
--- a/libtiff/tif_strip.c
+++ b/libtiff/tif_strip.c
@@ -1,4 +1,4 @@
-/* $Id: tif_strip.c,v 1.19.2.3 2010-12-15 00:50:30 faxguy Exp $ */
+/* $Id: tif_strip.c,v 1.19.2.4 2012-06-15 21:45:04 tgl Exp $ */
 
 /*
  * Copyright (c) 1991-1997 Sam Leffler
@@ -107,6 +107,7 @@ tsize_t
 TIFFVStripSize(TIFF* tif, uint32 nrows)
 {
 	TIFFDirectory *td = &tif->tif_dir;
+	uint32 stripsize;
 
 	if (nrows == (uint32) -1)
 		nrows = td->td_imagelength;
@@ -122,7 +123,7 @@ TIFFVStripSize(TIFF* tif, uint32 nrows)
 		 * YCbCr data for the extended image.
 		 */
 		uint16 ycbcrsubsampling[2];
-		tsize_t w, scanline, samplingarea;
+		uint32 w, scanline, samplingarea;
 
 		TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING,
 				      ycbcrsubsampling + 0,
@@ -141,13 +142,27 @@ TIFFVStripSize(TIFF* tif, uint32 nrows)
 		nrows = TIFFroundup(nrows, ycbcrsubsampling[1]);
 		/* NB: don't need TIFFhowmany here 'cuz everything is rounded */
 		scanline = multiply(tif, nrows, scanline, "TIFFVStripSize");
-		return ((tsize_t)
-		    summarize(tif, scanline,
-			      multiply(tif, 2, scanline / samplingarea,
-				       "TIFFVStripSize"), "TIFFVStripSize"));
+		/* a zero anywhere in here means overflow, must return zero */
+		if (scanline > 0) {
+			uint32 extra =
+			    multiply(tif, 2, scanline / samplingarea,
+				     "TIFFVStripSize");
+			if (extra > 0)
+				stripsize = summarize(tif, scanline, extra,
+						      "TIFFVStripSize");
+			else
+				stripsize = 0;
+		} else
+			stripsize = 0;
 	} else
-		return ((tsize_t) multiply(tif, nrows, TIFFScanlineSize(tif),
-					   "TIFFVStripSize"));
+		stripsize = multiply(tif, nrows, TIFFScanlineSize(tif),
+				     "TIFFVStripSize");
+	/* Because tsize_t is signed, we might have conversion overflow */
+	if (((tsize_t) stripsize) < 0) {
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", "TIFFVStripSize");
+		stripsize = 0;
+	}
+	return (tsize_t) stripsize;
 }
 
 
diff --git a/libtiff/tif_tile.c b/libtiff/tif_tile.c
index d8379e61..a3d959e6 100644
--- a/libtiff/tif_tile.c
+++ b/libtiff/tif_tile.c
@@ -1,4 +1,4 @@
-/* $Id: tif_tile.c,v 1.12.2.1 2010-06-08 18:50:43 bfriesen Exp $ */
+/* $Id: tif_tile.c,v 1.12.2.2 2012-06-15 21:45:04 tgl Exp $ */
 
 /*
  * Copyright (c) 1991-1997 Sam Leffler
@@ -174,7 +174,7 @@ tsize_t
 TIFFTileRowSize(TIFF* tif)
 {
 	TIFFDirectory *td = &tif->tif_dir;
-	tsize_t rowsize;
+	uint32 rowsize;
 	
 	if (td->td_tilelength == 0 || td->td_tilewidth == 0)
 		return ((tsize_t) 0);
@@ -193,7 +193,7 @@ tsize_t
 TIFFVTileSize(TIFF* tif, uint32 nrows)
 {
 	TIFFDirectory *td = &tif->tif_dir;
-	tsize_t tilesize;
+	uint32 tilesize;
 
 	if (td->td_tilelength == 0 || td->td_tilewidth == 0 ||
 	    td->td_tiledepth == 0)
@@ -209,12 +209,12 @@ TIFFVTileSize(TIFF* tif, uint32 nrows)
 		 * horizontal/vertical subsampling area include
 		 * YCbCr data for the extended image.
 		 */
-		tsize_t w =
+		uint32 w =
 		    TIFFroundup(td->td_tilewidth, td->td_ycbcrsubsampling[0]);
-		tsize_t rowsize =
+		uint32 rowsize =
 		    TIFFhowmany8(multiply(tif, w, td->td_bitspersample,
 					  "TIFFVTileSize"));
-		tsize_t samplingarea =
+		uint32 samplingarea =
 		    td->td_ycbcrsubsampling[0]*td->td_ycbcrsubsampling[1];
 		if (samplingarea == 0) {
 			TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Invalid YCbCr subsampling");
@@ -223,15 +223,27 @@ TIFFVTileSize(TIFF* tif, uint32 nrows)
 		nrows = TIFFroundup(nrows, td->td_ycbcrsubsampling[1]);
 		/* NB: don't need TIFFhowmany here 'cuz everything is rounded */
 		tilesize = multiply(tif, nrows, rowsize, "TIFFVTileSize");
-		tilesize = summarize(tif, tilesize,
-				     multiply(tif, 2, tilesize / samplingarea,
-					      "TIFFVTileSize"),
+		/* a zero anywhere in here means overflow, must return zero */
+		if (tilesize > 0) {
+			uint32 extra =
+			    multiply(tif, 2, tilesize / samplingarea,
 				     "TIFFVTileSize");
+			if (extra > 0)
+				tilesize = summarize(tif, tilesize, extra,
+						     "TIFFVTileSize");
+			else
+				tilesize = 0;
+		}
 	} else
 		tilesize = multiply(tif, nrows, TIFFTileRowSize(tif),
 				    "TIFFVTileSize");
-	return ((tsize_t)
-	    multiply(tif, tilesize, td->td_tiledepth, "TIFFVTileSize"));
+	tilesize = multiply(tif, tilesize, td->td_tiledepth, "TIFFVTileSize");
+	/* Because tsize_t is signed, we might have conversion overflow */
+	if (((tsize_t) tilesize) < 0) {
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", "TIFFVTileSize");
+		tilesize = 0;
+	}
+	return (tsize_t) tilesize;
 }
 
 /*
diff --git a/tools/tiff2pdf.c b/tools/tiff2pdf.c
index 7b1549e9..1c923477 100644
--- a/tools/tiff2pdf.c
+++ b/tools/tiff2pdf.c
@@ -1,4 +1,4 @@
-/* $Id: tiff2pdf.c,v 1.37.2.19 2010-12-13 05:41:11 faxguy Exp $
+/* $Id: tiff2pdf.c,v 1.37.2.20 2012-06-15 21:45:04 tgl Exp $
  *
  * tiff2pdf - converts a TIFF image to a PDF document
  *
@@ -431,6 +431,34 @@ t2p_unmapproc(thandle_t handle, tdata_t data, toff_t offset)
 	(void) handle, (void) data, (void) offset;
 }
 
+static uint64
+checkAdd64(uint64 summand1, uint64 summand2, T2P* t2p)
+{
+	uint64 bytes = summand1 + summand2;
+
+	if (bytes - summand1 != summand2) {
+		TIFFError(TIFF2PDF_MODULE, "Integer overflow");
+		t2p->t2p_error = T2P_ERR_ERROR;
+		bytes = 0;
+	}
+
+	return bytes;
+}
+
+static uint64
+checkMultiply64(uint64 first, uint64 second, T2P* t2p)
+{
+	uint64 bytes = first * second;
+
+	if (second && bytes / second != first) {
+		TIFFError(TIFF2PDF_MODULE, "Integer overflow");
+		t2p->t2p_error = T2P_ERR_ERROR;
+		bytes = 0;
+	}
+
+	return bytes;
+}
+
 /*
 
   This is the main function.
@@ -1773,9 +1801,7 @@ void t2p_read_tiff_size(T2P* t2p, TIFF* input){
 	tstrip_t i=0;
 	tstrip_t stripcount=0;
 #endif
-#ifdef OJPEG_SUPPORT
-        tsize_t k = 0;
-#endif
+        uint64 k = 0;
 
 	if(t2p->pdf_transcode == T2P_TRANSCODE_RAW){
 #ifdef CCITT_SUPPORT
@@ -1803,19 +1829,25 @@ void t2p_read_tiff_size(T2P* t2p, TIFF* input){
 			}
 			stripcount=TIFFNumberOfStrips(input);
 			for(i=0;i<stripcount;i++){
-				k += sbc[i];
+				k = checkAdd64(k, sbc[i], t2p);
 			}
 			if(TIFFGetField(input, TIFFTAG_JPEGIFOFFSET, &(t2p->tiff_dataoffset))){
 				if(t2p->tiff_dataoffset != 0){
 					if(TIFFGetField(input, TIFFTAG_JPEGIFBYTECOUNT, &(t2p->tiff_datasize))!=0){
 						if(t2p->tiff_datasize < k) {
-							t2p->pdf_ojpegiflength=t2p->tiff_datasize;
-							t2p->tiff_datasize+=k;
-							t2p->tiff_datasize+=6;
-							t2p->tiff_datasize+=2*stripcount;
 							TIFFWarning(TIFF2PDF_MODULE, 
 								"Input file %s has short JPEG interchange file byte count", 
 								TIFFFileName(input));
+							t2p->pdf_ojpegiflength=t2p->tiff_datasize;
+							k = checkAdd64(k, t2p->tiff_datasize, t2p);
+							k = checkAdd64(k, 6, t2p);
+							k = checkAdd64(k, stripcount, t2p);
+							k = checkAdd64(k, stripcount, t2p);
+							t2p->tiff_datasize = (tsize_t) k;
+							if ((uint64) t2p->tiff_datasize != k) {
+								TIFFError(TIFF2PDF_MODULE, "Integer overflow");
+								t2p->t2p_error = T2P_ERR_ERROR;
+							}
 							return;
 						}
 						return;
@@ -1828,9 +1860,14 @@ void t2p_read_tiff_size(T2P* t2p, TIFF* input){
 					}
 				}
 			}
-			t2p->tiff_datasize+=k;
-			t2p->tiff_datasize+=2*stripcount;
-			t2p->tiff_datasize+=2048;
+			k = checkAdd64(k, stripcount, t2p);
+			k = checkAdd64(k, stripcount, t2p);
+			k = checkAdd64(k, 2048, t2p);
+			t2p->tiff_datasize = (tsize_t) k;
+			if ((uint64) t2p->tiff_datasize != k) {
+				TIFFError(TIFF2PDF_MODULE, "Integer overflow");
+				t2p->t2p_error = T2P_ERR_ERROR;
+			}
 			return;
 		}
 #endif
@@ -1839,11 +1876,11 @@ void t2p_read_tiff_size(T2P* t2p, TIFF* input){
 			uint32 count = 0;
 			if(TIFFGetField(input, TIFFTAG_JPEGTABLES, &count, &jpt) != 0 ){
 				if(count > 4){
-					t2p->tiff_datasize += count;
-					t2p->tiff_datasize -= 2; /* don't use EOI of header */
+					k += count;
+					k -= 2; /* don't use EOI of header */
 				}
 			} else {
-				t2p->tiff_datasize = 2; /* SOI for first strip */
+				k = 2; /* SOI for first strip */
 			}
 			stripcount=TIFFNumberOfStrips(input);
 			if(!TIFFGetField(input, TIFFTAG_STRIPBYTECOUNTS, &sbc)){
@@ -1854,18 +1891,33 @@ void t2p_read_tiff_size(T2P* t2p, TIFF* input){
 				return;
 			}
 			for(i=0;i<stripcount;i++){
-				t2p->tiff_datasize += sbc[i];
-				t2p->tiff_datasize -=4; /* don't use SOI or EOI of strip */
+				k = checkAdd64(k, sbc[i], t2p);
+				k -=4; /* don't use SOI or EOI of strip */
+			}
+			k = checkAdd64(k, 2, t2p); /* use EOI of last strip */
+			t2p->tiff_datasize = (tsize_t) k;
+			if ((uint64) t2p->tiff_datasize != k) {
+				TIFFError(TIFF2PDF_MODULE, "Integer overflow");
+				t2p->t2p_error = T2P_ERR_ERROR;
 			}
-			t2p->tiff_datasize +=2; /* use EOI of last strip */
 			return;
 		}
 #endif
 		(void) 0;
 	}
-	t2p->tiff_datasize=TIFFScanlineSize(input) * t2p->tiff_length;
+	k = checkMultiply64(TIFFScanlineSize(input), t2p->tiff_length, t2p);
 	if(t2p->tiff_planar==PLANARCONFIG_SEPARATE){
-		t2p->tiff_datasize*= t2p->tiff_samplesperpixel;
+		k = checkMultiply64(k, t2p->tiff_samplesperpixel, t2p);
+	}
+	if (k == 0) {
+		/* Assume we had overflow inside TIFFScanlineSize */
+		t2p->t2p_error = T2P_ERR_ERROR;
+	}
+
+	t2p->tiff_datasize = (tsize_t) k;
+	if ((uint64) t2p->tiff_datasize != k) {
+		TIFFError(TIFF2PDF_MODULE, "Integer overflow");
+		t2p->t2p_error = T2P_ERR_ERROR;
 	}
 
 	return;
@@ -1883,6 +1935,7 @@ void t2p_read_tiff_size_tile(T2P* t2p, TIFF* input, ttile_t tile){
 #ifdef JPEG_SUPPORT
 	unsigned char* jpt;
 #endif
+        uint64 k;
 
 	edge |= t2p_tile_is_right_edge(t2p->tiff_tiles[t2p->pdf_page], tile);
 	edge |= t2p_tile_is_bottom_edge(t2p->tiff_tiles[t2p->pdf_page], tile);
@@ -1894,14 +1947,17 @@ void t2p_read_tiff_size_tile(T2P* t2p, TIFF* input, ttile_t tile){
 #endif
 		){
 			t2p->tiff_datasize=TIFFTileSize(input);
+			if (t2p->tiff_datasize == 0) {
+				/* Assume we had overflow inside TIFFTileSize */
+				t2p->t2p_error = T2P_ERR_ERROR;
+			}
 			return;
 		} else {
 			TIFFGetField(input, TIFFTAG_TILEBYTECOUNTS, &tbc);
-			t2p->tiff_datasize=tbc[tile];
+			k=tbc[tile];
 #ifdef OJPEG_SUPPORT
 			if(t2p->tiff_compression==COMPRESSION_OJPEG){
-				t2p->tiff_datasize+=2048;
-				return;
+			  	k = checkAdd64(k, 2048, t2p);
 			}
 #endif
 #ifdef JPEG_SUPPORT
@@ -1909,18 +1965,33 @@ void t2p_read_tiff_size_tile(T2P* t2p, TIFF* input, ttile_t tile){
 				uint32 count = 0;
 				if(TIFFGetField(input, TIFFTAG_JPEGTABLES, &count, &jpt)!=0){
 					if(count > 4){
-						t2p->tiff_datasize += count;
-						t2p->tiff_datasize -= 2; /* don't use EOI of header or SOI of tile */
+						k = checkAdd64(k, count, t2p);
+						k -= 2; /* don't use EOI of header or SOI of tile */
 					}
 				}
 			}
 #endif
+			t2p->tiff_datasize = (tsize_t) k;
+			if ((uint64) t2p->tiff_datasize != k) {
+				TIFFError(TIFF2PDF_MODULE, "Integer overflow");
+				t2p->t2p_error = T2P_ERR_ERROR;
+			}
 			return;
 		}
 	}
-	t2p->tiff_datasize=TIFFTileSize(input);
+	k = TIFFTileSize(input);
 	if(t2p->tiff_planar==PLANARCONFIG_SEPARATE){
-		t2p->tiff_datasize*= t2p->tiff_samplesperpixel;
+		k = checkMultiply64(k, t2p->tiff_samplesperpixel, t2p);
+	}
+	if (k == 0) {
+		/* Assume we had overflow inside TIFFTileSize */
+		t2p->t2p_error = T2P_ERR_ERROR;
+	}
+
+	t2p->tiff_datasize = (tsize_t) k;
+	if ((uint64) t2p->tiff_datasize != k) {
+		TIFFError(TIFF2PDF_MODULE, "Integer overflow");
+		t2p->t2p_error = T2P_ERR_ERROR;
 	}
 
 	return;
@@ -2013,6 +2084,10 @@ tsize_t t2p_readwrite_pdf_image(T2P* t2p, TIFF* input, TIFF* output){
 	uint32 max_striplength=0;
 #endif
 
+	/* Fail if prior error (in particular, can't trust tiff_datasize) */
+	if (t2p->t2p_error != T2P_ERR_OK)
+		return(0);
+
 	if(t2p->pdf_transcode == T2P_TRANSCODE_RAW){
 #ifdef CCITT_SUPPORT
 		if(t2p->pdf_compression == T2P_COMPRESS_G4){
@@ -2586,6 +2661,10 @@ tsize_t t2p_readwrite_pdf_image_tile(T2P* t2p, TIFF* input, TIFF* output, ttile_
 	uint32 xuint32=0;
 #endif
 
+	/* Fail if prior error (in particular, can't trust tiff_datasize) */
+	if (t2p->t2p_error != T2P_ERR_OK)
+		return(0);
+
 	edge |= t2p_tile_is_right_edge(t2p->tiff_tiles[t2p->pdf_page], tile);
 	edge |= t2p_tile_is_bottom_edge(t2p->tiff_tiles[t2p->pdf_page], tile);