jbig2dec: Fix signedness conversions.

5 files changed, 55 insertions, 31 deletions

diff --git a/jbig2dec/jbig2.c b/jbig2dec/jbig2.c
index 323ea29dd..f31c2dcc1 100644
--- a/jbig2dec/jbig2.c
+++ b/jbig2dec/jbig2.c
@@ -211,6 +211,22 @@ jbig2_get_uint32(const byte *bptr)
     return ((uint32_t) get_uint16(bptr) << 16) | get_uint16(bptr + 2);
 }
 
+static size_t
+jbig2_find_buffer_size(size_t desired)
+{
+    const size_t initial_buf_size = 1024;
+    size_t size = initial_buf_size;
+
+    if (desired == SIZE_MAX)
+        return SIZE_MAX;
+
+    while (size < desired)
+        size <<= 1;
+
+    return size;
+}
+
+
 /**
  * jbig2_data_in: submit data for decoding
  * @ctx: The jbig2dec decoder context
@@ -226,14 +242,8 @@ jbig2_get_uint32(const byte *bptr)
 int
 jbig2_data_in(Jbig2Ctx *ctx, const unsigned char *data, size_t size)
 {
-    const size_t initial_buf_size = 1024;
-
     if (ctx->buf == NULL) {
-        size_t buf_size = initial_buf_size;
-
-        do
-            buf_size <<= 1;
-        while (buf_size < size);
+        size_t buf_size = jbig2_find_buffer_size(size);
         ctx->buf = jbig2_new(ctx, byte, buf_size);
         if (ctx->buf == NULL) {
             return jbig2_error(ctx, JBIG2_SEVERITY_FATAL, JBIG2_UNKNOWN_SEGMENT_NUMBER, "failed to allocate buffer when reading data");
@@ -241,21 +251,26 @@ jbig2_data_in(Jbig2Ctx *ctx, const unsigned char *data, size_t size)
         ctx->buf_size = buf_size;
         ctx->buf_rd_ix = 0;
         ctx->buf_wr_ix = 0;
-    } else if (ctx->buf_wr_ix + size > ctx->buf_size) {
-        if (ctx->buf_rd_ix <= (ctx->buf_size >> 1) && ctx->buf_wr_ix - ctx->buf_rd_ix + size <= ctx->buf_size) {
-            memmove(ctx->buf, ctx->buf + ctx->buf_rd_ix, ctx->buf_wr_ix - ctx->buf_rd_ix);
+    } else if (size > ctx->buf_size - ctx->buf_wr_ix) {
+        size_t already = ctx->buf_wr_ix - ctx->buf_rd_ix;
+
+        if (ctx->buf_rd_ix <= (ctx->buf_size >> 1) && size <= ctx->buf_size - already) {
+            memmove(ctx->buf, ctx->buf + ctx->buf_rd_ix, already);
         } else {
             byte *buf;
-            size_t buf_size = initial_buf_size;
+            size_t buf_size;
+
+            if (already > SIZE_MAX - size) {
+                return jbig2_error(ctx, JBIG2_SEVERITY_FATAL, JBIG2_UNKNOWN_SEGMENT_NUMBER, "read data causes buffer to grow too large");
+            }
+
+            buf_size = jbig2_find_buffer_size(size + already);
 
-            do
-                buf_size <<= 1;
-            while (buf_size < ctx->buf_wr_ix - ctx->buf_rd_ix + size);
             buf = jbig2_new(ctx, byte, buf_size);
             if (buf == NULL) {
                 return jbig2_error(ctx, JBIG2_SEVERITY_FATAL, JBIG2_UNKNOWN_SEGMENT_NUMBER, "failed to allocate bigger buffer when reading data");
             }
-            memcpy(buf, ctx->buf + ctx->buf_rd_ix, ctx->buf_wr_ix - ctx->buf_rd_ix);
+            memcpy(buf, ctx->buf + ctx->buf_rd_ix, already);
             jbig2_free(ctx->allocator, ctx->buf);
             ctx->buf = buf;
             ctx->buf_size = buf_size;
@@ -263,6 +278,7 @@ jbig2_data_in(Jbig2Ctx *ctx, const unsigned char *data, size_t size)
         ctx->buf_wr_ix -= ctx->buf_rd_ix;
         ctx->buf_rd_ix = 0;
     }
+
     memcpy(ctx->buf + ctx->buf_wr_ix, data, size);
     ctx->buf_wr_ix += size;
 
@@ -387,7 +403,7 @@ jbig2_data_in(Jbig2Ctx *ctx, const unsigned char *data, size_t size)
                 segment->rows = jbig2_get_uint32(p);
                 p += 4;
 
-                segment->data_length = p - s;
+                segment->data_length = (size_t) (p - s);
                 jbig2_error(ctx, JBIG2_SEVERITY_INFO, segment->number, "unknown length determined to be %lu", (long) segment->data_length);
             }
             else if (segment->data_length > ctx->buf_wr_ix - ctx->buf_rd_ix)
@@ -483,15 +499,15 @@ jbig2_word_stream_buf_get_next_word(Jbig2Ctx *ctx, Jbig2WordStream *self, size_t
     }
 
     if (offset < z->size) {
-        val |= z->data[offset] << 24;
+        val = (uint32_t) z->data[offset] << 24;
         ret++;
     }
     if (offset + 1 < z->size) {
-        val |= z->data[offset + 1] << 16;
+        val |= (uint32_t) z->data[offset + 1] << 16;
         ret++;
     }
     if (offset + 2 < z->size) {
-        val |= z->data[offset + 2] << 8;
+        val |= (uint32_t) z->data[offset + 2] << 8;
         ret++;
     }
     if (offset + 3 < z->size) {
diff --git a/jbig2dec/jbig2_arith.c b/jbig2dec/jbig2_arith.c
index efda89793..b61353c81 100644
--- a/jbig2dec/jbig2_arith.c
+++ b/jbig2dec/jbig2_arith.c
@@ -36,10 +36,10 @@ struct _Jbig2ArithState {
     int CT;
 
     uint32_t next_word;
-    int next_word_bytes;
+    size_t next_word_bytes;
 
     Jbig2WordStream *ws;
-    int offset;
+    size_t offset;
 };
 
 /*
@@ -94,10 +94,12 @@ jbig2_arith_bytein(Jbig2Ctx *ctx, Jbig2ArithState *as)
 
         /* next_word_bytes can only be == 1 here, but let's be defensive. */
         if (as->next_word_bytes <= 1) {
-            as->next_word_bytes = as->ws->get_next_word(ctx, as->ws, as->offset, &as->next_word);
-            if (as->next_word_bytes < 0) {
+            int ret = as->ws->get_next_word(ctx, as->ws, as->offset, &as->next_word);
+            if (ret < 0) {
                 return jbig2_error(ctx, JBIG2_SEVERITY_WARNING, JBIG2_UNKNOWN_SEGMENT_NUMBER, "failed to check for marker code due to failure in underlying stream during arithmetic decoding");
             }
+
+            as->next_word_bytes = (size_t) ret;
             if (as->next_word_bytes == 0) {
                 jbig2_error(ctx, JBIG2_SEVERITY_WARNING, JBIG2_UNKNOWN_SEGMENT_NUMBER, "failed to read end of possible terminating marker code, assuming terminating marker code");
                 as->next_word = 0xFF900000;
@@ -151,10 +153,12 @@ jbig2_arith_bytein(Jbig2Ctx *ctx, Jbig2ArithState *as)
         as->next_word_bytes--;
 
         if (as->next_word_bytes == 0) {
-            as->next_word_bytes = as->ws->get_next_word(ctx, as->ws, as->offset, &as->next_word);
-            if (as->next_word_bytes < 0) {
+            int ret = as->ws->get_next_word(ctx, as->ws, as->offset, &as->next_word);
+            if (ret < 0) {
                 return jbig2_error(ctx, JBIG2_SEVERITY_WARNING, JBIG2_UNKNOWN_SEGMENT_NUMBER, "failed to read from underlying stream during arithmetic decoding");
             }
+            as->next_word_bytes = (size_t) ret;
+
             if (as->next_word_bytes == 0) {
                 jbig2_error(ctx, JBIG2_SEVERITY_WARNING, JBIG2_UNKNOWN_SEGMENT_NUMBER, "failed to find terminating marker code before end of underlying stream, assuming terminating marker code");
                 as->next_word = 0xFF900000;
@@ -183,6 +187,7 @@ Jbig2ArithState *
 jbig2_arith_new(Jbig2Ctx *ctx, Jbig2WordStream *ws)
 {
     Jbig2ArithState *result;
+    int ret;
 
     result = jbig2_new(ctx, Jbig2ArithState, 1);
     if (result == NULL) {
@@ -193,17 +198,20 @@ jbig2_arith_new(Jbig2Ctx *ctx, Jbig2WordStream *ws)
     result->ws = ws;
     result->offset = 0;
 
-    result->next_word_bytes = result->ws->get_next_word(ctx, result->ws, result->offset, &result->next_word);
-    if (result->next_word_bytes < 0) {
+    ret = result->ws->get_next_word(ctx, result->ws, result->offset, &result->next_word);
+    if (ret < 0) {
         jbig2_free(ctx->allocator, result);
         jbig2_error(ctx, JBIG2_SEVERITY_WARNING, JBIG2_UNKNOWN_SEGMENT_NUMBER, "failed to initialize underlying stream of arithmetic decoder");
         return NULL;
     }
+
+    result->next_word_bytes = (size_t) ret;
     if (result->next_word_bytes == 0) {
         jbig2_free(ctx->allocator, result);
         jbig2_error(ctx, JBIG2_SEVERITY_FATAL, JBIG2_UNKNOWN_SEGMENT_NUMBER, "failed to read first byte from underlying stream when initializing arithmetic decoder");
         return NULL;
     }
+
     result->offset += result->next_word_bytes;
 
     /* Figure F.1 */
diff --git a/jbig2dec/jbig2_arith_iaid.c b/jbig2dec/jbig2_arith_iaid.c
index 1baedcfc3..4d9fc2d84 100644
--- a/jbig2dec/jbig2_arith_iaid.c
+++ b/jbig2dec/jbig2_arith_iaid.c
@@ -53,7 +53,7 @@ jbig2_arith_iaid_ctx_new(Jbig2Ctx *ctx, uint8_t SBSYMCODELEN)
         return NULL;
     }
 
-    ctx_size = 1 << SBSYMCODELEN;
+    ctx_size = 1U << SBSYMCODELEN;
 
     result = jbig2_new(ctx, Jbig2ArithIaidCtx, 1);
     if (result == NULL) {
diff --git a/jbig2dec/jbig2_huffman.c b/jbig2dec/jbig2_huffman.c
index aa38ed2d5..689661898 100644
--- a/jbig2dec/jbig2_huffman.c
+++ b/jbig2dec/jbig2_huffman.c
@@ -178,7 +178,7 @@ jbig2_dump_huffman_table(const Jbig2HuffmanTable *table)
 int
 jbig2_huffman_skip(Jbig2HuffmanState *hs)
 {
-    int bits = hs->offset_bits & 7;
+    uint32_t bits = hs->offset_bits & 7;
     int code;
 
     if (bits) {
diff --git a/jbig2dec/jbig2_priv.h b/jbig2dec/jbig2_priv.h
index 08cd35734..6980e2d26 100644
--- a/jbig2dec/jbig2_priv.h
+++ b/jbig2dec/jbig2_priv.h
@@ -86,8 +86,8 @@ struct _Jbig2Ctx {
 
     byte *buf;
     size_t buf_size;
-    unsigned int buf_rd_ix;
-    unsigned int buf_wr_ix;
+    size_t buf_rd_ix;
+    size_t buf_wr_ix;
 
     Jbig2FileState state;