jbig2dec: Limit image size to 2GByte and check for overflow.

The width/height fields of a JBIG2 image are limited by the
specification to UINT32_MAX. This means that the total size of
the image is limited to UINT32_MAX * UINT32_MAX pixels.

jbig2_alloc() take a size argument of type size_t, limiting
allocations to at most SIZE_MAX bytes. For 32 bit systems
SIZE_MAX == UINT32_MAX and for 64 bit systems SIZE_MAX ==
UINT64_MAX.

This means that for 32 bit systems the maximum image size is
limited to UINT32_MAX * 8 pixels. For 64 bit systems it could
conceivably be limited to UINT64_MAX * 8 pixels if all indexing
into the image buffer allocation is 64 bit safe.

However jbig2dec's indexing into the image buffer allocations is
not always 64 bit safe, so limit image allocations to the safe
choice of 2Gbyte. This equates to limiting image sizes to
INT32_MAX * 8 pixels, or a square of 131072 by 131072 pixels.

1 files changed, 17 insertions, 14 deletions

diff --git a/jbig2dec/jbig2_image.c b/jbig2dec/jbig2_image.c
index 60137bdd9..fad5610b0 100644
--- a/jbig2dec/jbig2_image.c
+++ b/jbig2dec/jbig2_image.c
@@ -30,13 +30,16 @@
 #include "jbig2_priv.h"
 #include "jbig2_image.h"
 
+#if !defined (INT32_MAX)
+#define INT32_MAX  0x7fffffff
+#endif
+
 /* allocate a Jbig2Image structure and its associated bitmap */
 Jbig2Image *
 jbig2_image_new(Jbig2Ctx *ctx, uint32_t width, uint32_t height)
 {
     Jbig2Image *image;
     uint32_t stride;
-    int64_t check;
 
     if (width == 0 || height == 0) {
         jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1,
@@ -52,16 +55,16 @@ jbig2_image_new(Jbig2Ctx *ctx, uint32_t width, uint32_t height)
     }
 
     stride = ((width - 1) >> 3) + 1;    /* generate a byte-aligned stride */
+
     /* check for integer multiplication overflow */
-    check = ((int64_t) stride) * ((int64_t) height);
-    if (check != (int)check) {
-        jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1, "integer multiplication overflow from stride(%d)*height(%d)", stride, height);
+    if (height > (INT32_MAX / stride)) {
+        jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1, "integer multiplication overflow stride=%u, height=%u", stride, height);
         jbig2_free(ctx->allocator, image);
         return NULL;
     }
-    image->data = jbig2_new(ctx, uint8_t, (int)check);
+    image->data = jbig2_new(ctx, uint8_t, (size_t) height * stride);
     if (image->data == NULL) {
-        jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1, "could not allocate image data buffer! [stride(%d)*height(%d) bytes]", stride, height);
+        jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1, "could not allocate image data buffer! [stride(%u)*height(%u) bytes]", stride, height);
         jbig2_free(ctx->allocator, image);
         return NULL;
     }
@@ -110,21 +113,19 @@ jbig2_image_resize(Jbig2Ctx *ctx, Jbig2Image *image, uint32_t width, uint32_t he
 {
     if (width == image->width) {
         /* check for integer multiplication overflow */
-        int64_t check = ((int64_t) image->stride) * ((int64_t) height);
-
-        if (check != (int)check) {
-            jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1, "integer multiplication overflow during resize stride(%d)*height(%d)", image->stride, height);
+        if (image->height > (INT32_MAX / image->stride)) {
+            jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1, "integer multiplication overflow during resize stride(%u)*height(%u)", image->stride, height);
             return NULL;
         }
         /* use the same stride, just change the length */
-        image->data = jbig2_renew(ctx, image->data, uint8_t, (int)check);
+        image->data = jbig2_renew(ctx, image->data, uint8_t, (size_t) height * image->stride);
         if (image->data == NULL) {
             jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1, "could not resize image buffer");
             return NULL;
         }
         if (height > image->height) {
             const uint8_t fill = value ? 0xFF : 0x00;
-            memset(image->data + image->height * image->stride, fill, (height - image->height) * image->stride);
+            memset(image->data + (size_t) image->height * image->stride, fill, ((size_t) height - image->height) * image->stride);
         }
         image->height = height;
 
@@ -323,8 +324,10 @@ jbig2_image_compose(Jbig2Ctx *ctx, Jbig2Image *dst, Jbig2Image *src, int x, int
     /* general OR case */
     s = ss;
     d = dd = dst->data + y * dst->stride + leftbyte;
-    if (d < dst->data || leftbyte > dst->stride || d - leftbyte + h * dst->stride > dst->data + dst->height * dst->stride ||
-            s - leftbyte + (h - 1) * src->stride + rightbyte > src->data + src->height * src->stride) {
+    if (d < dst->data ||
+        leftbyte > dst->stride ||
+        d - leftbyte + (size_t) h * dst->stride > dst->data + (size_t) dst->height * dst->stride ||
+        s - leftbyte + (size_t) (h - 1) * src->stride + rightbyte > src->data + (size_t) src->height * src->stride) {
         return jbig2_error(ctx, JBIG2_SEVERITY_FATAL, -1, "preventing heap overflow in jbig2_image_compose");
     }
     if (leftbyte == rightbyte) {