Bug 703336: Fix GPL crash with jpg input on 64bit windows.

The Jpeg code uses setjmp/longjmp for error handling. On Windows at
least, in 64bit builds, jmpbufs need to be 16 byte aligned. This is
generally achieved by malloc blocks being 16 byte aligned, and
structures have their offsets calculated so that their contents lie
at appropriate positions.

Unfortunately, our malloc routines only align to 8 byte aligned.

We therefore bend the jpgtop.c code to align the jmpbuf manually.

Similar tricks are required in the pngtop.c code, which also uses
setjmp/longjmp, but a slightly different approach has to be taken
as the png routine allocates the structure with the callback in
itself, so we don't get to align just that. Instead, we modify the
allocation routines that we provide to pnglib so that blocks are
correctly padded.

2 files changed, 60 insertions, 8 deletions

diff --git a/gpdl/jpgtop.c b/gpdl/jpgtop.c
index 59d14f4b1..aa7b556ff 100644
--- a/gpdl/jpgtop.c
+++ b/gpdl/jpgtop.c
@@ -48,7 +48,7 @@ typedef enum
 
 typedef struct {
     struct jpeg_error_mgr pub;
-    jmp_buf setjmp_buffer;
+    jmp_buf *setjmp_buffer;
 } jpg_error_mgr;
 
 /*
@@ -91,6 +91,13 @@ typedef struct jpg_interp_instance_s {
     size_t             bytes_to_skip;
     jpg_error_mgr      jerr;
 
+    struct
+    {
+        jmp_buf setjmp_buffer;
+        unsigned char pad[16];
+    } aligned_jmpbuf;
+
+
     byte              *samples;
 
 } jpg_interp_instance_t;
@@ -412,7 +419,7 @@ jpg_error_exit(j_common_ptr cinfo)
 {
     jpg_error_mgr *jerr = (jpg_error_mgr *)cinfo->err;
 
-    longjmp(jerr->setjmp_buffer, 1);
+    longjmp(*jerr->setjmp_buffer, 1);
 }
 
 static int
@@ -457,6 +464,28 @@ consume_jpeg_data(jpg_interp_instance_t *jpg, stream_cursor_read *pr)
     return (pr->limit - pr->ptr == jpg->bytes_available_on_entry);
 }
 
+/* Horrible hack. Windows appears to expect the jmpbuf to be 16 byte
+ * aligned. Most 64bit platforms appear to return blocks from malloc
+ * that are aligned to 16 byte boundaries. Our malloc routines only
+ * return things to be 8 byte aligned on 64bit platforms, so we may
+ * fail. Accordingly, we allocate the jmpbuf in a larger structure,
+ * and align it.
+ *
+ * This is only a calculation we have to do once, so we just do it on
+ * all platforms. */
+static jmp_buf *
+align_setjmp_buffer(void *ptr)
+{
+    intptr_t offset = (intptr_t)ptr;
+    char *aligned;
+    offset &= 15;
+    offset = (16-offset) & 15;
+
+    aligned = ((char *)ptr) + offset;
+
+    return (jmp_buf *)(void *)aligned;
+}
+
 static int
 jpg_impl_process(pl_interp_implementation_t * impl, stream_cursor_read * pr)
 {
@@ -493,9 +522,11 @@ jpg_impl_process(pl_interp_implementation_t * impl, stream_cursor_read * pr)
                 break;
             }
 
+            jpg->jerr.setjmp_buffer = align_setjmp_buffer(&jpg->aligned_jmpbuf);
+
             jpg->cinfo.err = jpeg_std_error(&jpg->jerr.pub);
             jpg->jerr.pub.error_exit = jpg_error_exit;
-            if (setjmp(jpg->jerr.setjmp_buffer)) {
+            if (setjmp(*jpg->jerr.setjmp_buffer)) {
                 jpg->state = ii_state_flush;
                 break;
             }
@@ -521,7 +552,7 @@ jpg_impl_process(pl_interp_implementation_t * impl, stream_cursor_read * pr)
                 need_more_data = 1;
                 break; /* No bytes left after skipping */
             }
-            if (setjmp(jpg->jerr.setjmp_buffer)) {
+            if (setjmp(*jpg->jerr.setjmp_buffer)) {
                 jpeg_destroy_decompress(&jpg->cinfo);
                 jpg->state = ii_state_flush;
                 break;
@@ -657,7 +688,7 @@ jpg_impl_process(pl_interp_implementation_t * impl, stream_cursor_read * pr)
                 need_more_data = 1;
                 break; /* No bytes left after skipping */
             }
-            if (setjmp(jpg->jerr.setjmp_buffer)) {
+            if (setjmp(*jpg->jerr.setjmp_buffer)) {
                 jpeg_destroy_decompress(&jpg->cinfo);
                 jpg->state = ii_state_flush;
                 break;
@@ -678,7 +709,7 @@ jpg_impl_process(pl_interp_implementation_t * impl, stream_cursor_read * pr)
                 need_more_data = 1;
                 break; /* No bytes left after skipping */
             }
-            if (setjmp(jpg->jerr.setjmp_buffer)) {
+            if (setjmp(*jpg->jerr.setjmp_buffer)) {
                 jpeg_destroy_decompress(&jpg->cinfo);
                 jpg->state = ii_state_flush;
                 break;
@@ -720,7 +751,7 @@ jpg_impl_process(pl_interp_implementation_t * impl, stream_cursor_read * pr)
                 need_more_data = 1;
                 break; /* No bytes left after skipping */
             }
-            if (setjmp(jpg->jerr.setjmp_buffer)) {
+            if (setjmp(*jpg->jerr.setjmp_buffer)) {
                 jpeg_destroy_decompress(&jpg->cinfo);
                 jpg->state = ii_state_flush;
                 break;
@@ -735,7 +766,7 @@ jpg_impl_process(pl_interp_implementation_t * impl, stream_cursor_read * pr)
         }
         default:
         case ii_state_flush:
-            if (setjmp(jpg->jerr.setjmp_buffer))
+            if (setjmp(*jpg->jerr.setjmp_buffer))
                 break;
             jpeg_destroy_decompress(&jpg->cinfo);
 
diff --git a/gpdl/pngtop.c b/gpdl/pngtop.c
index 82e973fcd..f56d73518 100644
--- a/gpdl/pngtop.c
+++ b/gpdl/pngtop.c
@@ -327,6 +327,20 @@ my_png_malloc(png_structp png_ptr, png_alloc_size_t size)
 {
     png_interp_instance_t *png = (png_interp_instance_t *)png_get_mem_ptr(png_ptr);
 
+    if (sizeof(void *) == 8) {
+        /* gs_alloc_bytes returns blocks aligned to 8 on 64bit platforms.
+         * PNG (on Windows at least) requires blocks aligned to 16. */
+        unsigned char *block = gs_alloc_bytes(png->memory, size+16, "my_png_malloc");
+        intptr_t num_bytes_padded;
+
+        if (block == NULL)
+            return NULL;
+        num_bytes_padded = 16-(((intptr_t)block) & 15);
+        block += num_bytes_padded;
+        block[-1] = num_bytes_padded;
+
+        return block;
+    }
     return gs_alloc_bytes(png->memory, size, "my_png_malloc");
 }
 
@@ -335,6 +349,13 @@ my_png_free(png_structp png_ptr, png_voidp ptr)
 {
     png_interp_instance_t *png = (png_interp_instance_t *)png_get_mem_ptr(png_ptr);
 
+    if (sizeof(void *) == 8) {
+        unsigned char *block = ptr;
+        if (block == NULL)
+            return;
+        block -= block[-1];
+        ptr = (void *)block;
+    }
     gs_free_object(png->memory, ptr, "my_png_free");
 }