xpswrite - limit the memory used by images to avoid memory exhaustion

OSS-fuzz 53398 (and others)

Because xpswrite uses libtiff to write images to XPS files, and libtiff
won't offer us any way to use our memory manager, it is possible to
get a corrupted image which tries to use silly amounts of memory, which
leads to system memory being exhausted.

In this commit; change the gs_memory_status_s structure to include the
'limit' from the memory allocator. Alter the various allocators to fill
in the value (or set it to -1 if there is no limit). In the xpswrite
device calculate the amount of memory needed for an uncompressed image,
if it becomes negative we know we broke a 64-bit signed integer, so
abort. If it didn't go negative, and the memory manager has a specified
limit, check to see if the image will fit in that. If not return
VMerror.

This prevents at least some of the memory exhaustion errors with
xpswrite and OSS-fuzz, because OSS-fuzz sets -K to limit memory use.

6 files changed, 37 insertions, 3 deletions

diff --git a/base/gsalloc.c b/base/gsalloc.c
index 15162a27f..394c63b37 100644
--- a/base/gsalloc.c
+++ b/base/gsalloc.c
@@ -1786,6 +1786,7 @@ i_status(gs_memory_t * mem, gs_memory_status_t * pstat)
     pstat->allocated = imem->allocated +
         imem->previous_status.allocated;
     pstat->max_used = 0;		/* unknown for this allocator */
+    pstat->limit = imem->limit;
     pstat->is_thread_safe = false;	/* this allocator is not thread safe */
 }
 
diff --git a/base/gsmalloc.c b/base/gsmalloc.c
index 63c8b6bd6..6aad18c17 100644
--- a/base/gsmalloc.c
+++ b/base/gsmalloc.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2001-2021 Artifex Software, Inc.
+/* Copyright (C) 2001-2022 Artifex Software, Inc.
    All Rights Reserved.
 
    This software is provided AS-IS with no warranty, either express or
@@ -465,6 +465,7 @@ gs_heap_status(gs_memory_t * mem, gs_memory_status_t * pstat)
     pstat->allocated = mmem->used + avail_snapshot;
     pstat->used = mmem->used;
     pstat->max_used = mmem->max_used;
+    pstat->limit = mmem->limit;
     pstat->is_thread_safe = true;	/* this allocator has a mutex (monitor) and IS thread safe */
     if (mmem->monitor)
         gx_monitor_leave(mmem->monitor);	/* Done with exclusive access */
diff --git a/base/gsmalloc.h b/base/gsmalloc.h
index a4b10f008..2d5a5b5c1 100644
--- a/base/gsmalloc.h
+++ b/base/gsmalloc.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2001-2021 Artifex Software, Inc.
+/* Copyright (C) 2001-2022 Artifex Software, Inc.
    All Rights Reserved.
 
    This software is provided AS-IS with no warranty, either express or
diff --git a/base/gsmchunk.c b/base/gsmchunk.c
index 5d3dbebe1..32d56a118 100644
--- a/base/gsmchunk.c
+++ b/base/gsmchunk.c
@@ -1362,6 +1362,7 @@ chunk_status(gs_memory_t * mem, gs_memory_status_t * pstat)
     pstat->allocated = cmem->used;
     pstat->used = cmem->used - cmem->total_free;
     pstat->max_used = cmem->max_used;
+    pstat->limit = (size_t)~1;      /* No limit on allocations */
     pstat->is_thread_safe = false;	/* this allocator does not have an internal mutex */
 }
 
diff --git a/base/gsmemory.h b/base/gsmemory.h
index 5be297ea3..33c1469d9 100644
--- a/base/gsmemory.h
+++ b/base/gsmemory.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2001-2021 Artifex Software, Inc.
+/* Copyright (C) 2001-2022 Artifex Software, Inc.
    All Rights Reserved.
 
    This software is provided AS-IS with no warranty, either express or
@@ -77,6 +77,8 @@ typedef struct gs_memory_status_s {
      * allocated data, space available for allocation, and overhead.
      */
     size_t allocated;
+    /* Set by -K for non-gc memory, max memory to use. */
+    size_t limit;
     /*
      * "Used" space is the amount of space used by allocated data
      * plus overhead.
diff --git a/devices/vector/gdevxps.c b/devices/vector/gdevxps.c
index d0f02f5d0..af0b32710 100644
--- a/devices/vector/gdevxps.c
+++ b/devices/vector/gdevxps.c
@@ -2102,6 +2102,35 @@ xps_begin_typed_image(gx_device               *dev,
 
     num_components = gs_color_space_num_components(pcs);
     bits_per_pixel = pim->BitsPerComponent * num_components;
+
+    {
+        /* This is a really hacky attempt to avoid running out of memory. This is
+         * inspired by various OSS-fuzz bugs but in particular 53398. This device
+         * uses the libtiff library to write images into the XPS file, libtiff won't
+         * let us use our own memory manager and have rejected patches to allow it
+         * to do so, so it uses system memory. If we have a badly broken file we
+         * might end up asking it to write a ridiculously large image which will
+         * cause it to eat all system memory. So here we try to limit it to any pre-defined
+         * limit. Also, if the width * height * 24 bits (RGB) goes negative then we know
+         * we've exceeded 2^(64 - 1) bytes, which is unreasonable too.
+         */
+        int64_t memory_needed = (int64_t)pim->Width * (int64_t)pim->Height * 3;
+        gs_memory_status_t status;
+
+        if (memory_needed < 0) {
+            gs_free_object(mem, pie, "xps_begin_image");
+            return_error(gs_error_VMerror);
+        }
+
+        gs_memory_status(dev->memory->gs_lib_ctx->memory, &status);
+        if (status.limit < (size_t)~1) {
+            if ( memory_needed > status.limit) {
+                gs_free_object(mem, pie, "xps_begin_image");
+                return_error(gs_error_VMerror);
+            }
+        }
+    }
+
     pie->decode_st.bps = bits_per_pixel / num_components;
     pie->bytes_comp = (pie->decode_st.bps > 8 ? 2 : 1);
     pie->decode_st.spp = num_components;