* libtiff/tif_read.c: TIFFFillStrip() and TIFFFillTile():

avoid excessive memory allocation in case of shorten files.
Only effective on 64 bit builds and non-mapped cases.
Credit to OSS-Fuzz (locally run, on GDAL)

2 files changed, 191 insertions, 13 deletions

diff --git a/ChangeLog b/ChangeLog
index bbba7ddb..34f99bd2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
 2017-05-10 Even Rouault <even.rouault at spatialys.com>
 
+	* libtiff/tif_read.c: TIFFFillStrip() and TIFFFillTile():
+	avoid excessive memory allocation in case of shorten files.
+	Only effective on 64 bit builds and non-mapped cases.
+	Credit to OSS-Fuzz (locally run, on GDAL)
+
+2017-05-10 Even Rouault <even.rouault at spatialys.com>
+
 	* libtiff/tif_zip.c, tif_pixarlog.c, tif_predict.c: fix memory
 	leak when the underlying codec (ZIP, PixarLog) succeeds its
 	setupdecode() method, but PredictorSetup fails.
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index 763fb97a..4dd3f486 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -1,4 +1,4 @@
-/* $Id: tif_read.c,v 1.54 2017-05-10 13:37:19 erouault Exp $ */
+/* $Id: tif_read.c,v 1.55 2017-05-10 19:38:49 erouault Exp $ */
 
 /*
  * Copyright (c) 1988-1997 Sam Leffler
@@ -463,6 +463,140 @@ TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
 	return (size);
 }
 
+#define INITIAL_THRESHOLD (1024 * 1024)
+#define THRESHOLD_MULTIPLIER 10
+#define MAX_THRESHOLD (THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * INITIAL_THRESHOLD)
+
+static tmsize_t
+TIFFReadRawStripOrTile2(TIFF* tif, uint32 strip_or_tile, int is_strip,
+                        tmsize_t size, const char* module)
+{
+#if SIZEOF_VOIDP == 8 || SIZEOF_SIZE_T == 8
+        tmsize_t threshold = INITIAL_THRESHOLD;
+#endif
+        tmsize_t already_read = 0;
+        TIFFDirectory *td = &tif->tif_dir;
+
+        assert( !isMapped(tif) );
+        assert((tif->tif_flags&TIFF_NOREADRAW)==0);
+
+        if (!SeekOK(tif, td->td_stripoffset[strip_or_tile])) {
+            if( is_strip )
+            {
+                TIFFErrorExt(tif->tif_clientdata, module,
+                    "Seek error at scanline %lu, strip %lu",
+                    (unsigned long) tif->tif_row,
+                    (unsigned long) strip_or_tile);
+            }
+            else
+            {
+                TIFFErrorExt(tif->tif_clientdata, module,
+                    "Seek error at row %lu, col %lu, tile %lu",
+                    (unsigned long) tif->tif_row,
+                    (unsigned long) tif->tif_col,
+                    (unsigned long) strip_or_tile);
+            }
+            return ((tmsize_t)(-1));
+        }
+
+        /* On 64 bit processes, read first a maximum of 1 MB, then 10 MB, etc */
+        /* so as to avoid allocating too much memory in case the file is too */
+        /* short. We could ask for the file size, but this might be */
+        /* expensive with some I/O layers (think of reading a gzipped file) */
+        /* Restrict to 64 bit processes, so as to avoid reallocs() */
+        /* on 32 bit processes where virtual memory is scarce.  */
+        while( already_read < size )
+        {
+            tmsize_t read;
+            tmsize_t to_read = size - already_read;
+#if SIZEOF_VOIDP == 8 || SIZEOF_SIZE_T == 8
+            if( to_read >= threshold && threshold < MAX_THRESHOLD &&
+                already_read + to_read > tif->tif_rawdatasize )
+            {
+                to_read = threshold;
+                threshold *= THRESHOLD_MULTIPLIER;
+            }
+#endif
+            if (already_read + to_read > tif->tif_rawdatasize) {
+                uint8* new_rawdata;
+                assert((tif->tif_flags & TIFF_MYBUFFER) != 0);
+                tif->tif_rawdatasize = (tmsize_t)TIFFroundup_64(
+                                    (uint64)already_read + to_read, 1024);
+                if (tif->tif_rawdatasize==0) {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                                "Invalid buffer size");
+                    return ((tmsize_t)(-1));
+                }
+                new_rawdata = (uint8*) _TIFFrealloc(
+                                tif->tif_rawdata, tif->tif_rawdatasize);
+                if( new_rawdata == 0 )
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                        "No space for data buffer at scanline %lu",
+                        (unsigned long) tif->tif_row);
+                    _TIFFfree(tif->tif_rawdata);
+                    tif->tif_rawdata = 0;
+                    tif->tif_rawdatasize = 0;
+                    return ((tmsize_t)(-1));
+                }
+                tif->tif_rawdata = new_rawdata;
+            }
+
+            read = TIFFReadFile(tif, tif->tif_rawdata + already_read, to_read);
+            already_read += read;
+            if (read != to_read) {
+                memset( tif->tif_rawdata + already_read, 0,
+                        tif->tif_rawdatasize - already_read );
+#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+                if( is_strip )
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                        "Read error at scanline %lu; got %I64u bytes, "
+                        "expected %I64u",
+                                        (unsigned long) tif->tif_row,
+                                        (unsigned __int64) already_read,
+                                        (unsigned __int64) size);
+                }
+                else
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                        "Read error at row %lu, col %lu, tile %lu; "
+                        "got %I64u bytes, expected %I64u",
+                                        (unsigned long) tif->tif_row,
+                                        (unsigned long) tif->tif_col,
+                                        (unsigned long) strip_or_tile,
+                                        (unsigned __int64) already_read,
+                                        (unsigned __int64) size);
+                }
+#else
+                if( is_strip )
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                        "Read error at scanline %lu; got %llu bytes, "
+                        "expected %llu",
+                                        (unsigned long) tif->tif_row,
+                                        (unsigned long long) already_read,
+                                        (unsigned long long) size);
+                }
+                else
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                        "Read error at row %lu, col %lu, tile %lu; "
+                        "got %llu bytes, expected %llu",
+                                        (unsigned long) tif->tif_row,
+                                        (unsigned long) tif->tif_col,
+                                        (unsigned long) strip_or_tile,
+                                        (unsigned long long) already_read,
+                                        (unsigned long long) size);
+                }
+#endif
+                return ((tmsize_t)(-1));
+            }
+        }
+
+        return (size);
+}
+
 /*
  * Read a strip of data from the file.
  */
@@ -666,17 +800,36 @@ TIFFFillStrip(TIFF* tif, uint32 strip)
 					    (unsigned long) strip);
 					return (0);
 				}
-				if (!TIFFReadBufferSetup(tif, 0, bytecountm))
-					return (0);
 			}
 			if (tif->tif_flags&TIFF_BUFFERMMAP) {
 				tif->tif_curstrip = NOSTRIP;
-				if (!TIFFReadBufferSetup(tif, 0, bytecountm))
+				tif->tif_rawdata = NULL;
+				tif->tif_rawdatasize = 0;
+				tif->tif_flags &= ~TIFF_BUFFERMMAP;
+			}
+
+			if( isMapped(tif) )
+			{
+				if (bytecountm > tif->tif_rawdatasize &&
+				    !TIFFReadBufferSetup(tif, 0, bytecountm))
+				{
+					return (0);
+				}
+				if (TIFFReadRawStrip1(tif, strip, tif->tif_rawdata,
+				    bytecountm, module) != bytecountm)
+				{
 					return (0);
+				}
 			}
-			if (TIFFReadRawStrip1(tif, strip, tif->tif_rawdata,
-				bytecountm, module) != bytecountm)
-				return (0);
+			else
+			{
+				if (TIFFReadRawStripOrTile2(tif, strip, 1,
+				    bytecountm, module) != bytecountm)
+				{
+					return (0);
+				}
+			}
+
 
                         tif->tif_rawdataoff = 0;
                         tif->tif_rawdataloaded = bytecountm;
@@ -959,18 +1112,36 @@ TIFFFillTile(TIFF* tif, uint32 tile)
 					    (unsigned long) tile);
 					return (0);
 				}
-				if (!TIFFReadBufferSetup(tif, 0, bytecountm))
-					return (0);
 			}
 			if (tif->tif_flags&TIFF_BUFFERMMAP) {
 				tif->tif_curtile = NOTILE;
-				if (!TIFFReadBufferSetup(tif, 0, bytecountm))
+				tif->tif_rawdata = NULL;
+				tif->tif_rawdatasize = 0;
+				tif->tif_flags &= ~TIFF_BUFFERMMAP;
+			}
+
+			if( isMapped(tif) )
+			{
+				if (bytecountm > tif->tif_rawdatasize &&
+				    !TIFFReadBufferSetup(tif, 0, bytecountm))
+				{
 					return (0);
+				}
+				if (TIFFReadRawTile1(tif, tile, tif->tif_rawdata,
+				    bytecountm, module) != bytecountm)
+				{
+					return (0);
+				}
+			}
+			else
+			{
+				if (TIFFReadRawStripOrTile2(tif, tile, 0,
+				    bytecountm, module) != bytecountm)
+				{
+					return (0);
+				}
 			}
 
-			if (TIFFReadRawTile1(tif, tile, tif->tif_rawdata,
-			    bytecountm, module) != bytecountm)
-				return (0);
 
                         tif->tif_rawdataoff = 0;
                         tif->tif_rawdataloaded = bytecountm;