jbig2dec: Avoid underflow in custom allocator.

1 files changed, 22 insertions, 29 deletions

diff --git a/jbig2dec/jbig2dec.c b/jbig2dec/jbig2dec.c
index 1a45e71dc..3996e5b17 100644
--- a/jbig2dec/jbig2dec.c
+++ b/jbig2dec/jbig2dec.c
@@ -98,7 +98,10 @@ static void *jbig2dec_alloc(Jbig2Allocator *allocator_, size_t size)
 
     if (size == 0)
         return NULL;
-    if (size > allocator->memory_limit - ALIGNMENT - allocator->memory_used)
+    if (size > SIZE_MAX - ALIGNMENT)
+        return NULL;
+
+    if (size + ALIGNMENT > allocator->memory_limit - allocator->memory_used)
         return NULL;
 
     ptr = malloc(size + ALIGNMENT);
@@ -136,42 +139,32 @@ static void jbig2dec_free(Jbig2Allocator *allocator_, void *p)
 static void *jbig2dec_realloc(Jbig2Allocator *allocator_, void *p, size_t size)
 {
     jbig2dec_allocator_t *allocator = (jbig2dec_allocator_t *) allocator_;
-    unsigned char *oldp = p ? (unsigned char *) p - ALIGNMENT : NULL;
+    unsigned char *oldp;
+    size_t oldsize;
 
-    if (size > SIZE_MAX - ALIGNMENT)
+    if (p == NULL)
+        return jbig2dec_alloc(allocator_, size);
+    if (p < (void *) ALIGNMENT)
         return NULL;
 
-    if (oldp == NULL)
-    {
-        if (size == 0)
-            return NULL;
-        if (size > allocator->memory_limit - ALIGNMENT - allocator->memory_used)
-            return NULL;
-
-        p = malloc(size + ALIGNMENT);
+    if (size == 0) {
+        jbig2dec_free(allocator_, p);
+        return NULL;
     }
-    else
-    {
-        size_t oldsize;
-        memcpy(&oldsize, oldp, sizeof(oldsize));
-
-        if (size == 0)
-        {
-            allocator->memory_used -= oldsize + ALIGNMENT;
-            free(oldp);
-            return NULL;
-        }
+    if (size > SIZE_MAX - ALIGNMENT)
+        return NULL;
 
-        if (size > allocator->memory_limit - allocator->memory_used + oldsize)
-            return NULL;
+    oldp = (unsigned char *) p - ALIGNMENT;
+    memcpy(&oldsize, oldp, sizeof(oldsize));
 
-        p = realloc(oldp, size + ALIGNMENT);
-        if (p == NULL)
-            return NULL;
+    if (size + ALIGNMENT > allocator->memory_limit - allocator->memory_used + oldsize + ALIGNMENT)
+        return NULL;
 
-        allocator->memory_used -= oldsize + ALIGNMENT;
-    }
+    p = realloc(oldp, size + ALIGNMENT);
+    if (p == NULL)
+        return NULL;
 
+    allocator->memory_used -= oldsize + ALIGNMENT;
     memcpy(p, &size, sizeof(size));
     allocator->memory_used += size + ALIGNMENT;