Emit explicit error message when tif_max_single_mem_alloc is exceeded

2 files changed, 65 insertions, 9 deletions

diff --git a/libtiff/tif_open.c b/libtiff/tif_open.c
index 76158096..a169de10 100644
--- a/libtiff/tif_open.c
+++ b/libtiff/tif_open.c
@@ -99,23 +99,37 @@ void TIFFOpenOptionsSetWarningHandlerExtR(TIFFOpenOptions* opts, TIFFErrorHandle
     opts->warnhandler_user_data = warnhandler_user_data;
 }
 
+static void _TIFFEmitErrorAboveMaxSingleMemAlloc(TIFF* tif, const char* pszFunction, tmsize_t s)
+{
+    TIFFErrorExtR(tif, pszFunction,
+                  "Memory allocation of %" PRIu64 " bytes is beyond the %" PRIu64 " byte limit defined in open options",
+                  (uint64_t)s,
+                  (uint64_t)tif->tif_max_single_mem_alloc);
+}
+
 /** malloc() version that takes into account memory-specific open options */
 void* _TIFFmallocExt(TIFF* tif, tmsize_t s)
 {
     if (tif != NULL && tif->tif_max_single_mem_alloc > 0 && s > tif->tif_max_single_mem_alloc)
+    {
+        _TIFFEmitErrorAboveMaxSingleMemAlloc(tif, "_TIFFmallocExt", s);
         return NULL;
+    }
     return _TIFFmalloc(s);
 }
 
 /** calloc() version that takes into account memory-specific open options */
 void* _TIFFcallocExt(TIFF* tif, tmsize_t nmemb, tmsize_t siz)
 {
-    if( tif != NULL )
+    if (tif != NULL && tif->tif_max_single_mem_alloc > 0)
     {
         if (nmemb <= 0 || siz <= 0 || nmemb > TIFF_TMSIZE_T_MAX / siz)
             return NULL;
-        if (tif->tif_max_single_mem_alloc > 0 && nmemb * siz > tif->tif_max_single_mem_alloc)
+        if (nmemb * siz > tif->tif_max_single_mem_alloc)
+        {
+            _TIFFEmitErrorAboveMaxSingleMemAlloc(tif, "_TIFFcallocExt", nmemb * siz);
             return NULL;
+        }
     }
     return _TIFFcalloc(nmemb, siz);
 }
@@ -124,7 +138,10 @@ void* _TIFFcallocExt(TIFF* tif, tmsize_t nmemb, tmsize_t siz)
 void* _TIFFreallocExt(TIFF* tif, void* p, tmsize_t s)
 {
     if (tif != NULL && tif->tif_max_single_mem_alloc > 0 && s > tif->tif_max_single_mem_alloc)
+    {
+        _TIFFEmitErrorAboveMaxSingleMemAlloc(tif, "_TIFFreallocExt", s);
         return NULL;
+    }
     return _TIFFrealloc(p, s);
 }
 
@@ -209,7 +226,7 @@ TIFFClientOpenExt(
 	tmsize_t size_to_alloc = (tmsize_t)(sizeof (TIFF) + strlen(name) + 1);
 	if (opts && opts->max_single_mem_alloc > 0 &&
 	    size_to_alloc > opts->max_single_mem_alloc) {
-		_TIFFErrorEarly(opts, clientdata, module, "%s: Out of memory (TIFF structure)", name);
+		_TIFFErrorEarly(opts, clientdata, module, "%s: Memory allocation of %" PRIu64 " bytes is beyond the %" PRIu64 " byte limit defined in open options", name, size_to_alloc, opts->max_single_mem_alloc);
 		goto bad2;
 	}
 	tif = (TIFF *)_TIFFmallocExt(NULL, size_to_alloc);
diff --git a/test/test_open_options.c b/test/test_open_options.c
index fbd32e64..182aeec9 100644
--- a/test/test_open_options.c
+++ b/test/test_open_options.c
@@ -39,6 +39,7 @@
 #endif
 
 #include "tiffio.h"
+#include "tiffiop.h" // for struct TIFF
 
 #define ERROR_STRING_SIZE 1024
 
@@ -132,7 +133,9 @@ static int test_error_handler()
     return ret;
 }
 
-static int test_TIFFOpenOptionsSetMaxSingleMemAlloc(tmsize_t limit)
+static int test_TIFFOpenOptionsSetMaxSingleMemAlloc(tmsize_t limit,
+                                                    int expected_to_fail_in_open,
+                                                    int expected_to_fail_in_write_directory)
 {
     int ret = 0;
     TIFFOpenOptions* opts = TIFFOpenOptionsAlloc();
@@ -140,11 +143,45 @@ static int test_TIFFOpenOptionsSetMaxSingleMemAlloc(tmsize_t limit)
     TIFFOpenOptionsSetMaxSingleMemAlloc(opts, limit);
     TIFF* tif = TIFFOpenExt("test_error_handler.tif", "w", opts);
     TIFFOpenOptionsFree(opts);
-    if( tif != NULL )
+    if (expected_to_fail_in_open)
     {
-        fprintf(stderr, "Expected TIFFOpenExt() to fail due to memory limitation\n");
-        ret = 1;
-        TIFFClose(tif);
+        if (tif != NULL)
+        {
+            fprintf(stderr, "Expected TIFFOpenExt() to fail due to memory limitation\n");
+            ret = 1;
+            TIFFClose(tif);
+        }
+    }
+    else
+    {
+        if (tif == NULL)
+        {
+            fprintf(stderr, "Expected TIFFOpenExt() to succeed\n");
+            ret = 1;
+        }
+        else
+        {
+#define VALUE_SAMPLESPERPIXEL 10000
+            TIFFSetField(tif, TIFFTAG_SAMPLESPERPIXEL, VALUE_SAMPLESPERPIXEL);
+            TIFFSetField(tif, TIFFTAG_BITSPERSAMPLE, 8);
+            if (TIFFWriteDirectory(tif) == 0)
+            {
+                if (!expected_to_fail_in_write_directory)
+                {
+                    fprintf(stderr, "Expected TIFFWriteDirectory() to succeed\n");
+                    ret = 1;
+                }
+            }
+            else
+            {
+                if (expected_to_fail_in_write_directory)
+                {
+                    fprintf(stderr, "Expected TIFFWriteDirectory() to fail\n");
+                    ret = 1;
+                }
+            }
+            TIFFClose(tif);
+        }
     }
     unlink("test_error_handler.tif");
     return ret;
@@ -154,6 +191,8 @@ int main()
 {
     int ret = 0;
     ret += test_error_handler();
-    ret += test_TIFFOpenOptionsSetMaxSingleMemAlloc(1);
+    ret += test_TIFFOpenOptionsSetMaxSingleMemAlloc(1, TRUE, -1);
+    ret += test_TIFFOpenOptionsSetMaxSingleMemAlloc(sizeof(TIFF) + strlen("test_error_handler.tif") + 1, FALSE, TRUE);
+    ret += test_TIFFOpenOptionsSetMaxSingleMemAlloc(VALUE_SAMPLESPERPIXEL * sizeof(short), FALSE, FALSE);
     return ret;
 }