tiffcrop correctly update buffersize after rotateImage() fix#520 -- enlarge buffsize and check integer overflow within rotateImage().

1 files changed, 33 insertions, 3 deletions

diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index f8b66188..ca23529b 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -9560,7 +9560,8 @@ static int rotateImage(uint16_t rotation, struct image_data *image,
     uint32_t bytes_per_pixel, bytes_per_sample;
     uint32_t row, rowsize, src_offset, dst_offset;
     uint32_t i, col, width, length;
-    uint32_t colsize, buffsize, col_offset, pix_offset;
+    uint32_t colsize, col_offset, pix_offset;
+    tmsize_t buffsize;
     unsigned char *ibuff;
     unsigned char *src;
     unsigned char *dst;
@@ -9573,12 +9574,40 @@ static int rotateImage(uint16_t rotation, struct image_data *image,
     spp = image->spp;
     bps = image->bps;
 
+    if ((spp != 0 && bps != 0 &&
+         width > (uint32_t)((UINT32_MAX - 7) / spp / bps)) ||
+        (spp != 0 && bps != 0 &&
+         length > (uint32_t)((UINT32_MAX - 7) / spp / bps)))
+    {
+        TIFFError("rotateImage", "Integer overflow detected.");
+        return (-1);
+    }
     rowsize = ((bps * spp * width) + 7) / 8;
     colsize = ((bps * spp * length) + 7) / 8;
     if ((colsize * width) > (rowsize * length))
-        buffsize = (colsize + 1) * width;
+    {
+        if (((tmsize_t)colsize + 1) != 0 &&
+            (tmsize_t)width > ((TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES) /
+                               ((tmsize_t)colsize + 1)))
+        {
+            TIFFError("rotateImage",
+                      "Integer overflow when calculating buffer size.");
+            return (-1);
+        }
+        buffsize = ((tmsize_t)colsize + 1) * width;
+    }
     else
+    {
+        if (((tmsize_t)rowsize + 1) != 0 &&
+            (tmsize_t)length > ((TIFF_TMSIZE_T_MAX - NUM_BUFF_OVERSIZE_BYTES) /
+                                ((tmsize_t)rowsize + 1)))
+        {
+            TIFFError("rotateImage",
+                      "Integer overflow when calculating buffer size.");
+            return (-1);
+        }
         buffsize = (rowsize + 1) * length;
+    }
 
     bytes_per_sample = (bps + 7) / 8;
     bytes_per_pixel = ((bps * spp) + 7) / 8;
@@ -9607,7 +9636,8 @@ static int rotateImage(uint16_t rotation, struct image_data *image,
               (unsigned char *)limitMalloc(buffsize + NUM_BUFF_OVERSIZE_BYTES)))
     {
         TIFFError("rotateImage",
-                  "Unable to allocate rotation buffer of %1u bytes",
+                  "Unable to allocate rotation buffer of %" TIFF_SSIZE_FORMAT
+                  " bytes ",
                   buffsize + NUM_BUFF_OVERSIZE_BYTES);
         return (-1);
     }