Changes to libtiff for gpdl.

1) Ensure that libtiff doesn't mess with 'boolean' in GS builds
on Windows. Without this, the jpeg structures used by our JPEG
lib build are different in size when called from gs and libtiff,
resulting in runtime errors.

2) Update libtiff so that it can correctly call into the jpeg
library so that memory operations happen from our pools, not
malloc/free. Slightly horrid in that this is more complex with
OJPEG than JPEG files.

5 files changed, 153 insertions, 23 deletions

diff --git a/tiff/libtiff/tif_jpeg.c b/tiff/libtiff/tif_jpeg.c
index 93ae2ead1..d49b90402 100644
--- a/tiff/libtiff/tif_jpeg.c
+++ b/tiff/libtiff/tif_jpeg.c
@@ -65,6 +65,10 @@ int TIFFJPEGIsFullStripRequired_12(TIFF* tif);
 # define XMD_H 1
 #endif
 
+/* If we are building for GS, do NOT mess with boolean - we want it to be int on all platforms.
+ */
+#define GS_TIFF_BUILD
+#ifndef GS_TIFF_BUILD
 /*
    The windows RPCNDR.H file defines boolean, but defines it with the
    unsigned char size.  You should compile JPEG library using appropriate
@@ -85,6 +89,7 @@ int TIFFJPEGIsFullStripRequired_12(TIFF* tif);
 # endif
 # define HAVE_BOOLEAN            /* prevent jmorecfg.h from redefining it */
 #endif
+#endif
 
 #include "jpeglib.h"
 #include "jerror.h"
@@ -290,15 +295,19 @@ TIFFjpeg_create_compress(JPEGState* sp)
 }
 
 static int
-TIFFjpeg_create_decompress(JPEGState* sp)
+TIFFjpeg_create_decompress(JPEGState* sp, TIFF *tif)
 {
 	/* initialize JPEG error handling */
 	sp->cinfo.d.err = jpeg_std_error(&sp->err);
 	sp->err.error_exit = TIFFjpeg_error_exit;
 	sp->err.output_message = TIFFjpeg_output_message;
 
-	/* set client_data to avoid UMR warning from tools like Purify */
-	sp->cinfo.d.client_data = NULL;
+        /* GS extension */
+	if (tif->get_jpeg_mem_ptr)
+		sp->cinfo.d.client_data = tif->get_jpeg_mem_ptr(tif->tif_clientdata);
+	else
+		/* set client_data to avoid UMR warning from tools like Purify */
+		sp->cinfo.d.client_data = NULL;
 
 	return CALLVJPEG(sp, jpeg_create_decompress(&sp->cinfo.d));
 }
@@ -1065,7 +1074,7 @@ int TIFFJPEGIsFullStripRequired(TIFF* tif)
     memset(&state, 0, sizeof(JPEGState));
     state.tif = tif;
 
-    TIFFjpeg_create_decompress(&state);
+    TIFFjpeg_create_decompress(&state, tif);
 
     TIFFjpeg_data_src(&state);
 
@@ -2443,7 +2452,7 @@ static int JPEGInitializeLibJPEG( TIFF * tif, int decompress )
      * Initialize libjpeg.
      */
     if ( decompress ) {
-        if (!TIFFjpeg_create_decompress(sp))
+        if (!TIFFjpeg_create_decompress(sp, tif))
             return (0);
     } else {
         if (!TIFFjpeg_create_compress(sp))
diff --git a/tiff/libtiff/tif_ojpeg.c b/tiff/libtiff/tif_ojpeg.c
index bf0d1a2a0..a3d4513a5 100644
--- a/tiff/libtiff/tif_ojpeg.c
+++ b/tiff/libtiff/tif_ojpeg.c
@@ -204,6 +204,10 @@ static const TIFFField ojpegFields[] = {
 # define XMD_H 1
 #endif
 
+/* If we are building for GS, do NOT mess with boolean - we want it to be int on all platforms.
+ */
+#define GS_TIFF_BUILD
+#ifndef GS_TIFF_BUILD
 /* Define "boolean" as unsigned char, not int, per Windows custom. */
 #if defined(__WIN32__) && !defined(__MINGW32__)
 # ifndef __RPCNDR_H__            /* don't conflict if rpcndr.h already read */
@@ -211,9 +215,13 @@ static const TIFFField ojpegFields[] = {
 # endif
 # define HAVE_BOOLEAN            /* prevent jmorecfg.h from redefining it */
 #endif
+#endif
 
 #include "jpeglib.h"
 #include "jerror.h"
+#ifdef GS_TIFF_BUILD
+#include "jmemcust.h"
+#endif
 
 typedef struct jpeg_error_mgr jpeg_error_mgr;
 typedef struct jpeg_common_struct jpeg_common_struct;
@@ -242,8 +250,8 @@ typedef enum {
 
 typedef struct {
 	TIFF* tif;
-        int decoder_ok;
-        int error_in_raw_data_decoding;
+	int decoder_ok;
+	int error_in_raw_data_decoding;
 	#ifndef LIBJPEG_ENCAP_EXTERNAL
 	JMP_BUF exit_jmpbuf;
 	#endif
@@ -334,6 +342,10 @@ typedef struct {
 	OJPEGStateOutState out_state;
 	uint8 out_buffer[OJPEG_BUFFER];
 	uint8* skip_buffer;
+#ifdef GS_TIFF_BUILD
+	jpeg_cust_mem_data jmem;
+	jpeg_cust_mem_data *jmem_parent;
+#endif
 } OJPEGState;
 
 static int OJPEGVGetField(TIFF* tif, uint32 tag, va_list ap);
@@ -423,7 +435,7 @@ TIFFInitOJPEG(TIFF* tif, int scheme)
 
 	assert(scheme==COMPRESSION_OJPEG);
 
-        /*
+	/*
 	 * Merge codec-specific tag information.
 	 */
 	if (!_TIFFMergeFields(tif, ojpegFields, TIFFArrayCount(ojpegFields))) {
@@ -1194,6 +1206,91 @@ OJPEGReadSecondarySos(TIFF* tif, uint16 s)
 	return(1);
 }
 
+#ifdef GS_TIFF_BUILD
+#define TIFF_FROM_CINFO(cinfo) \
+	((TIFF *)GET_CUST_MEM_DATA(cinfo)->priv)
+
+static void *j_mem_get_small(j_common_ptr cinfo, size_t size)
+{
+	jpeg_cust_mem_data *jc = GET_CUST_MEM_DATA(cinfo);
+	TIFF *tif = (TIFF *)jc->priv;
+	OJPEGState* sp=(OJPEGState*)tif->tif_data;
+	void *ret;
+
+	cinfo->client_data = sp->jmem_parent;
+	ret = sp->jmem_parent->j_mem_get_small(cinfo, size);
+	cinfo->client_data = jc;
+
+	return ret;
+}
+
+static void *j_mem_get_large(j_common_ptr cinfo, size_t size)
+{
+	jpeg_cust_mem_data *jc = GET_CUST_MEM_DATA(cinfo);
+	TIFF *tif = (TIFF *)jc->priv;
+	OJPEGState* sp=(OJPEGState*)tif->tif_data;
+	void *ret;
+
+	cinfo->client_data = sp->jmem_parent;
+	ret = sp->jmem_parent->j_mem_get_large(cinfo, size);
+	cinfo->client_data = jc;
+
+	return ret;
+}
+
+static void j_mem_free_small(j_common_ptr cinfo, void *object, size_t size)
+{
+	jpeg_cust_mem_data *jc = GET_CUST_MEM_DATA(cinfo);
+	TIFF *tif = (TIFF *)jc->priv;
+	OJPEGState* sp=(OJPEGState*)tif->tif_data;
+
+	cinfo->client_data = sp->jmem_parent;
+	sp->jmem_parent->j_mem_free_small(cinfo, object, size);
+	cinfo->client_data = jc;
+}
+
+static void j_mem_free_large(j_common_ptr cinfo, void *object, size_t size)
+{
+	jpeg_cust_mem_data *jc = GET_CUST_MEM_DATA(cinfo);
+	TIFF *tif = (TIFF *)jc->priv;
+	OJPEGState* sp=(OJPEGState*)tif->tif_data;
+
+	cinfo->client_data = sp->jmem_parent;
+	sp->jmem_parent->j_mem_free_large(cinfo, object, size);
+	cinfo->client_data = jc;
+}
+
+static long j_mem_init (j_common_ptr cinfo)
+{
+	jpeg_cust_mem_data *jc = GET_CUST_MEM_DATA(cinfo);
+	TIFF *tif = (TIFF *)jc->priv;
+	OJPEGState* sp=(OJPEGState*)tif->tif_data;
+	long ret;
+
+	cinfo->client_data = sp->jmem_parent;
+	ret = sp->jmem_parent->j_mem_init(cinfo);
+	cinfo->client_data = jc;
+
+	return ret;
+}
+
+static void j_mem_term (j_common_ptr cinfo)
+{
+	jpeg_cust_mem_data *jc = GET_CUST_MEM_DATA(cinfo);
+	TIFF *tif = (TIFF *)jc->priv;
+	OJPEGState* sp=(OJPEGState*)tif->tif_data;
+
+	cinfo->client_data = sp->jmem_parent;
+	sp->jmem_parent->j_mem_term(cinfo);
+	cinfo->client_data = jc;
+}
+#else
+
+#define TIFF_FROM_CINFO(cinfo) \
+	((TIFF *)GET_CUST_MEM_DATA(cinfo))
+
+#endif
+
 static int
 OJPEGWriteHeaderInfo(TIFF* tif)
 {
@@ -1210,7 +1307,17 @@ OJPEGWriteHeaderInfo(TIFF* tif)
 	sp->libjpeg_jpeg_error_mgr.output_message=OJPEGLibjpegJpegErrorMgrOutputMessage;
 	sp->libjpeg_jpeg_error_mgr.error_exit=OJPEGLibjpegJpegErrorMgrErrorExit;
 	sp->libjpeg_jpeg_decompress_struct.err=&(sp->libjpeg_jpeg_error_mgr);
+	/* set client_data to avoid UMR warning from tools like Purify */
+#ifdef GS_TIFF_BUILD
+	sp->jmem_parent = tif->get_jpeg_mem_ptr(tif->tif_clientdata);
+	(void)jpeg_cust_mem_init(&sp->jmem, (void *)tif,
+				 j_mem_init, j_mem_term, NULL,
+				 j_mem_get_small, j_mem_free_small,
+				 j_mem_get_large, j_mem_free_large, NULL);
+	sp->libjpeg_jpeg_decompress_struct.client_data=&sp->jmem;
+#else
 	sp->libjpeg_jpeg_decompress_struct.client_data=(void*)tif;
+#endif
 	if (jpeg_create_decompress_encap(sp,&(sp->libjpeg_jpeg_decompress_struct))==0)
 		return(0);
 	sp->libjpeg_session_active=1;
@@ -2531,18 +2638,20 @@ jpeg_encap_unwind(TIFF* tif)
 static void
 OJPEGLibjpegJpegErrorMgrOutputMessage(jpeg_common_struct* cinfo)
 {
+        TIFF *tif = TIFF_FROM_CINFO(cinfo);
 	char buffer[JMSG_LENGTH_MAX];
 	(*cinfo->err->format_message)(cinfo,buffer);
-	TIFFWarningExt(((TIFF*)(cinfo->client_data))->tif_clientdata,"LibJpeg","%s",buffer);
+	TIFFWarningExt(tif->tif_clientdata,"LibJpeg","%s",buffer);
 }
 
 static void
 OJPEGLibjpegJpegErrorMgrErrorExit(jpeg_common_struct* cinfo)
 {
+        TIFF *tif = TIFF_FROM_CINFO(cinfo);
 	char buffer[JMSG_LENGTH_MAX];
 	(*cinfo->err->format_message)(cinfo,buffer);
-	TIFFErrorExt(((TIFF*)(cinfo->client_data))->tif_clientdata,"LibJpeg","%s",buffer);
-	jpeg_encap_unwind((TIFF*)(cinfo->client_data));
+	TIFFErrorExt(tif->tif_clientdata,"LibJpeg","%s",buffer);
+	jpeg_encap_unwind(tif);
 }
 
 static void
@@ -2554,7 +2663,7 @@ OJPEGLibjpegJpegSourceMgrInitSource(jpeg_decompress_struct* cinfo)
 static boolean
 OJPEGLibjpegJpegSourceMgrFillInputBuffer(jpeg_decompress_struct* cinfo)
 {
-	TIFF* tif=(TIFF*)cinfo->client_data;
+        TIFF *tif = TIFF_FROM_CINFO(cinfo);
 	OJPEGState* sp=(OJPEGState*)tif->tif_data;
 	void* mem=0;
 	uint32 len=0U;
@@ -2571,7 +2680,7 @@ OJPEGLibjpegJpegSourceMgrFillInputBuffer(jpeg_decompress_struct* cinfo)
 static void
 OJPEGLibjpegJpegSourceMgrSkipInputData(jpeg_decompress_struct* cinfo, long num_bytes)
 {
-	TIFF* tif=(TIFF*)cinfo->client_data;
+        TIFF *tif = TIFF_FROM_CINFO(cinfo);
 	(void)num_bytes;
 	TIFFErrorExt(tif->tif_clientdata,"LibJpeg","Unexpected error");
 	jpeg_encap_unwind(tif);
@@ -2584,7 +2693,7 @@ OJPEGLibjpegJpegSourceMgrSkipInputData(jpeg_decompress_struct* cinfo, long num_b
 static boolean
 OJPEGLibjpegJpegSourceMgrResyncToRestart(jpeg_decompress_struct* cinfo, int desired)
 {
-	TIFF* tif=(TIFF*)cinfo->client_data;
+        TIFF *tif = TIFF_FROM_CINFO(cinfo);
 	(void)desired;
 	TIFFErrorExt(tif->tif_clientdata,"LibJpeg","Unexpected error");
 	jpeg_encap_unwind(tif);
diff --git a/tiff/libtiff/tif_open.c b/tiff/libtiff/tif_open.c
index 3cb53d4e6..ffe7835f3 100644
--- a/tiff/libtiff/tif_open.c
+++ b/tiff/libtiff/tif_open.c
@@ -731,6 +731,14 @@ TIFFGetUnmapFileProc(TIFF* tif)
 	return (tif->tif_unmapproc);
 }
 
+void
+TIFFSetJpegMemFunction(TIFF *tif,
+                       void *(*fn)(thandle_t))
+{
+    tif->get_jpeg_mem_ptr = fn;
+}
+
+
 /* vim: set ts=8 sts=8 sw=8 noet: */
 /*
  * Local Variables:
diff --git a/tiff/libtiff/tiffio.h b/tiff/libtiff/tiffio.h
index 198481d53..a626a7643 100644
--- a/tiff/libtiff/tiffio.h
+++ b/tiff/libtiff/tiffio.h
@@ -414,6 +414,9 @@ extern int TIFFRewriteDirectory(TIFF *);
 extern int TIFFDeferStrileArrayWriting(TIFF *);
 extern int TIFFForceStrileArrayWriting(TIFF* );
 
+/* Extra function to allow JPEG memory use to be controlled */
+extern void TIFFSetJpegMemFunction(TIFF *, void *(*fn)(thandle_t));
+
 #if defined(c_plusplus) || defined(__cplusplus)
 extern void TIFFPrintDirectory(TIFF*, FILE*, long = 0);
 extern int TIFFReadScanline(TIFF* tif, void* buf, uint32 row, uint16 sample = 0);
diff --git a/tiff/libtiff/tiffiop.h b/tiff/libtiff/tiffiop.h
index 5da1a51c4..900f47867 100644
--- a/tiff/libtiff/tiffiop.h
+++ b/tiff/libtiff/tiffiop.h
@@ -2,23 +2,23 @@
  * Copyright (c) 1988-1997 Sam Leffler
  * Copyright (c) 1991-1997 Silicon Graphics, Inc.
  *
- * Permission to use, copy, modify, distribute, and sell this software and 
+ * Permission to use, copy, modify, distribute, and sell this software and
  * its documentation for any purpose is hereby granted without fee, provided
  * that (i) the above copyright notices and this permission notice appear in
  * all copies of the software and related documentation, and (ii) the names of
  * Sam Leffler and Silicon Graphics may not be used in any advertising or
  * publicity relating to the software without the specific, prior written
  * permission of Sam Leffler and Silicon Graphics.
- * 
- * THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND, 
- * EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY 
- * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.  
- * 
+ *
+ * THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ *
  * IN NO EVENT SHALL SAM LEFFLER OR SILICON GRAPHICS BE LIABLE FOR
  * ANY SPECIAL, INCIDENTAL, INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND,
  * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
- * WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF 
- * LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE 
+ * WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF
+ * LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  * OF THIS SOFTWARE.
  */
 
@@ -61,7 +61,7 @@
 #ifdef HAVE_ASSERT_H
 # include <assert.h>
 #else
-# define assert(x) 
+# define assert(x)
 #endif
 
 #ifdef HAVE_SEARCH_H
@@ -240,6 +240,7 @@ struct tiff {
 	 * setting up an old tag extension scheme. */
 	TIFFFieldArray*      tif_fieldscompat;
 	size_t               tif_nfieldscompat;
+	void                *(*get_jpeg_mem_ptr)(thandle_t);
 };
 
 #define isPseudoTag(t) (t > 0xffff)            /* is tag value normal or pseudo */