Merge branch 'libdeflate' into 'master'

Add support for building against libdeflate for faster Zip/Deflate compression/decompression

See merge request libtiff/libtiff!158

16 files changed, 444 insertions, 21 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c8a89403..7efe145b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
-image: ubuntu:16.04
+image: ubuntu:20.04
+
 before_script:
-  - apt-get update -qq && apt-get install -y -qq autoconf automake build-essential cmake libtool libjpeg8-dev libjbig-dev liblzma-dev ninja-build zlib1g-dev zip wget
+  - apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq autoconf automake build-essential cmake libtool libjpeg8-dev libjbig-dev liblzma-dev ninja-build zlib1g-dev libdeflate-dev zip wget
 
 stages:
   - build
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a1f964da..6669b706 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -457,6 +457,33 @@ if(ZLIB_FOUND)
   set(ZLIB_SUPPORT 1)
 endif()
 set(ZIP_SUPPORT ${ZLIB_SUPPORT})
+
+
+# libdeflate
+option(libdeflate "use libdeflate (optional for faster Deflate support, still requires zlib)" ON)
+if (libdeflate)
+  set(DEFLATE_FOUND 0)
+  find_path(DEFLATE_INCLUDE_DIR libdeflate.h)
+  set(DEFLATE_NAMES ${DEFLATE_NAMES} libdeflate)
+  find_library(DEFLATE_LIBRARY NAMES ${DEFLATE_NAMES})
+  if (DEFLATE_INCLUDE_DIR AND DEFLATE_LIBRARY)
+    set(DEFLATE_FOUND 1)
+    set(DEFLATE_LIBRARIES ${DEFLATE_LIBRARY})
+  endif()
+endif()
+set(LIBDEFLATE_SUPPORT FALSE)
+if(DEFLATE_FOUND)
+  set(LIBDEFLATE_SUPPORT TRUE)
+endif()
+
+if(LIBDEFLATE_SUPPORT AND NOT ZIP_SUPPORT)
+    message(WARNING "libdeflate available but zlib is not. libdeflate cannot be used")
+    set(LIBDEFLATE_SUPPORT FALSE)
+endif()
+
+set(LIBDEFLATE_SUPPORT ${LIBDEFLATE_SUPPORT})
+
+
 # Option for Pixar log-format algorithm
 
 # Pixar log format
@@ -655,6 +682,9 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libtiff-4.pc
 if(ZLIB_INCLUDE_DIRS)
   list(APPEND TIFF_INCLUDES ${ZLIB_INCLUDE_DIRS})
 endif()
+if(DEFLATE_INCLUDE_DIR)
+  list(APPEND TIFF_INCLUDES ${DEFLATE_INCLUDE_DIR})
+endif()
 if(JPEG_INCLUDE_DIR)
   list(APPEND TIFF_INCLUDES ${JPEG_INCLUDE_DIR})
 endif()
@@ -682,6 +712,9 @@ endif()
 if(ZLIB_LIBRARIES)
   list(APPEND TIFF_LIBRARY_DEPS ${ZLIB_LIBRARIES})
 endif()
+if(DEFLATE_LIBRARIES)
+  list(APPEND TIFF_LIBRARY_DEPS ${DEFLATE_LIBRARIES})
+endif()
 if(JPEG_LIBRARIES)
   list(APPEND TIFF_LIBRARY_DEPS ${JPEG_LIBRARIES})
 endif()
@@ -737,6 +770,7 @@ message(STATUS "  LogLuv high dynamic range encoding: ${logluv}")
 message(STATUS "")
 message(STATUS " Support for external codecs:")
 message(STATUS "  ZLIB support:                       ${zlib} (requested) ${ZLIB_FOUND} (availability)")
+message(STATUS "  libdeflate support:                 ${libdeflate} (requested) ${LIBDEFLATE_SUPPORT} (availability)")
 message(STATUS "  Pixar log-format algorithm:         ${pixarlog} (requested) ${PIXARLOG_SUPPORT} (availability)")
 message(STATUS "  JPEG support:                       ${jpeg} (requested) ${JPEG_FOUND} (availability)")
 message(STATUS "  Old JPEG support:                   ${old-jpeg} (requested) ${JPEG_FOUND} (availability)")
diff --git a/configure.ac b/configure.ac
index eecb8e99..cfba5118 100644
--- a/configure.ac
+++ b/configure.ac
@@ -615,6 +615,63 @@ if test "$HAVE_ZLIB" = "yes" ; then
 fi
 
 dnl ---------------------------------------------------------------------------
+dnl Check for libdeflate.
+dnl ---------------------------------------------------------------------------
+
+HAVE_LIBDEFLATE=no
+
+AC_ARG_ENABLE(libdeflate,
+	      AS_HELP_STRING([--disable-libdeflate],
+			     [disable libdeflate usage (optional for faster Deflate support (still requires zlib), enabled by default)]),,)
+AC_ARG_WITH(libdeflate-include-dir,
+	    AS_HELP_STRING([--with-libdeflate-include-dir=DIR],
+			   [location of libdeflate headers]),,)
+AC_ARG_WITH(libdeflate-lib-dir,
+	    AS_HELP_STRING([--with-libdeflate-lib-dir=DIR],
+			   [location of libdeflate library binary]),,)
+
+if test "x$enable_libdeflate" != "xno" ; then
+
+  if test "x$with_libdeflate_lib_dir" != "x" ; then
+    LDFLAGS="-L$with_libdeflate_lib_dir $LDFLAGS"
+  fi
+
+  AC_CHECK_LIB(deflate, libdeflate_zlib_decompress, [libdeflate_lib=yes], [libdeflate_lib=no],)
+  if test "$libdeflate_lib" = "no" -a "x$with_libdeflate_lib_dir" != "x"; then
+    AC_MSG_ERROR([libdeflate library not found at $with_libdeflate_lib_dir])
+  fi
+
+  if test "x$with_libdeflate_include_dir" != "x" ; then
+    CPPFLAGS="-I$with_libdeflate_include_dir $CPPFLAGS"
+  fi
+  AC_CHECK_HEADER(libdeflate.h, [libdeflate_h=yes], [libdeflate_h=no])
+  if test "$libdeflate_h" = "no" -a "x$with_libdeflate_include_dir" != "x" ; then
+    AC_MSG_ERROR([libdeflate headers not found at $with_libdeflate_include_dir])
+  fi
+
+  if test "$libdeflate_lib" = "yes" -a "$libdeflate_h" = "yes" ; then
+    HAVE_LIBDEFLATE=yes
+  fi
+
+fi
+
+if test "$HAVE_LIBDEFLATE" = "yes" -a "$HAVE_ZLIB" = "no" ; then
+  AC_MSG_WARN([libdeflate available but zlib is not. libdeflate cannot be used])
+  HAVE_LIBDEFLATE=no
+fi  
+
+if test "$HAVE_LIBDEFLATE" = "yes" ; then
+  AC_DEFINE(LIBDEFLATE_SUPPORT,1,[Support libdeflate enhanced compression])
+  LIBS="-ldeflate $LIBS"
+  tiff_libs_private="-ldeflate ${tiff_libs_private}"
+
+  if test "$HAVE_RPATH" = "yes" -a "x$with_libdeflate_lib_dir" != "x" ; then
+    LIBDIR="-R $with_libdeflate_lib_dir $LIBDIR"
+  fi
+
+fi
+
+dnl ---------------------------------------------------------------------------
 dnl Check for Pixar log-format algorithm.
 dnl ---------------------------------------------------------------------------
 
@@ -1191,6 +1248,7 @@ LOC_MSG([  LogLuv high dynamic range encoding: ${HAVE_LOGLUV}])
 LOC_MSG()
 LOC_MSG([ Support for external codecs:])
 LOC_MSG([  ZLIB support:                       ${HAVE_ZLIB}])
+LOC_MSG([  libdeflate support:                 ${HAVE_LIBDEFLATE}])
 LOC_MSG([  Pixar log-format algorithm:         ${HAVE_PIXARLOG}])
 LOC_MSG([  JPEG support:                       ${HAVE_JPEG}])
 LOC_MSG([  Old JPEG support:                   ${HAVE_OJPEG}])
diff --git a/libtiff/tif_write.c b/libtiff/tif_write.c
index f79330e9..3af69ab4 100644
--- a/libtiff/tif_write.c
+++ b/libtiff/tif_write.c
@@ -668,6 +668,10 @@ TIFFWriteBufferSetup(TIFF* tif, void* bp, tmsize_t size)
 	if (size == (tmsize_t)(-1)) {
 		size = (isTiled(tif) ?
 		    tif->tif_tilesize : TIFFStripSize(tif));
+
+                /* Adds 10% margin for cases where compression would expand a bit */
+                if( size < TIFF_TMSIZE_T_MAX - size / 10 )
+                    size += size / 10;
 		/*
 		 * Make raw data buffer at least 8K
 		 */
diff --git a/libtiff/tif_zip.c b/libtiff/tif_zip.c
index d8fe919b..0e6e3bf3 100644
--- a/libtiff/tif_zip.c
+++ b/libtiff/tif_zip.c
@@ -29,24 +29,22 @@
  *
  * ZIP (aka Deflate) Compression Support
  *
- * This file is simply an interface to the zlib library written by
+ * This file is an interface to the zlib library written by
  * Jean-loup Gailly and Mark Adler.  You must use version 1.0 or later
- * of the library: this code assumes the 1.0 API and also depends on
- * the ability to write the zlib header multiple times (one per strip)
- * which was not possible with versions prior to 0.95.  Note also that
- * older versions of this codec avoided this bug by suppressing the header
- * entirely.  This means that files written with the old library cannot
- * be read; they should be converted to a different compression scheme
- * and then reconverted.
+ * of the library.
  *
- * The data format used by the zlib library is described in the files
- * zlib-3.1.doc, deflate-1.1.doc and gzip-4.1.doc, available in the
- * directory ftp://ftp.uu.net/pub/archiving/zip/doc.  The library was
- * last found at ftp://ftp.uu.net/pub/archiving/zip/zlib/zlib-0.99.tar.gz.
+ * Optionnaly, libdeflate (https://github.com/ebiggers/libdeflate) may be used
+ * to do the compression and decompression, but only for whole strips and tiles.
+ * For scanline access, zlib will be sued as a fallback.
  */
 #include "tif_predict.h"
 #include "zlib.h"
 
+#if LIBDEFLATE_SUPPORT
+#include "libdeflate.h"
+#endif
+#define LIBDEFLATE_MAX_COMPRESSION_LEVEL 12
+
 #include <stdio.h>
 
 /*
@@ -70,6 +68,12 @@ typedef struct {
         z_stream        stream;
 	int             zipquality;            /* compression level */
 	int             state;                 /* state flags */
+	int             subcodec;              /* DEFLATE_SUBCODEC_ZLIB or DEFLATE_SUBCODEC_LIBDEFLATE */
+#if LIBDEFLATE_SUPPORT
+	int             libdeflate_state;       /* -1 = until first time ZIPEncode() / ZIPDecode() is called, 0 = use zlib, 1 = use libdeflate */
+	struct libdeflate_decompressor* libdeflate_dec;
+	struct libdeflate_compressor*   libdeflate_enc;
+#endif
 #define ZSTATE_INIT_DECODE 0x01
 #define ZSTATE_INIT_ENCODE 0x02
 
@@ -132,6 +136,9 @@ ZIPPreDecode(TIFF* tif, uint16 s)
 	if( (sp->state & ZSTATE_INIT_DECODE) == 0 )
             tif->tif_setupdecode( tif );
 
+#if LIBDEFLATE_SUPPORT
+        sp->libdeflate_state = -1;
+#endif
 	sp->stream.next_in = tif->tif_rawdata;
 	assert(sizeof(sp->stream.avail_in)==4);  /* if this assert gets raised,
 	    we need to simplify this code to reflect a ZLib that is likely updated
@@ -151,6 +158,77 @@ ZIPDecode(TIFF* tif, uint8* op, tmsize_t occ, uint16 s)
 	assert(sp != NULL);
 	assert(sp->state == ZSTATE_INIT_DECODE);
 
+#if LIBDEFLATE_SUPPORT
+        if( sp->libdeflate_state == 1 )
+            return 0;
+
+        /* If we have libdeflate support and we are asked to read a whole */
+        /* strip/tile, then go for using it */
+        do {
+            TIFFDirectory *td = &tif->tif_dir;
+
+            if( sp->libdeflate_state == 0 )
+                break;
+            if( sp->subcodec == DEFLATE_SUBCODEC_ZLIB )
+                break;
+
+            /* Check if we are in the situation where we can use libdeflate */
+            if (isTiled(tif)) {
+                if( TIFFTileSize64(tif) != (uint64)occ )
+                    break;
+            } else {
+                uint32 strip_height = td->td_imagelength - tif->tif_row;
+                if (strip_height > td->td_rowsperstrip)
+                    strip_height = td->td_rowsperstrip;
+                if( TIFFVStripSize64(tif, strip_height) != (uint64)occ )
+                    break;
+            }
+
+            /* Check for overflow */
+            if( (size_t)tif->tif_rawcc != (uint64)tif->tif_rawcc )
+                break;
+            if( (size_t)occ != (uint64)occ )
+                break;
+
+            /* Go for decompression using libdeflate */
+            {
+                enum libdeflate_result res;
+                if( sp->libdeflate_dec == NULL )
+                {
+                    sp->libdeflate_dec = libdeflate_alloc_decompressor();
+                    if( sp->libdeflate_dec == NULL )
+                    {
+                        break;
+                    }
+                }
+
+                sp->libdeflate_state = 1;
+
+                res = libdeflate_zlib_decompress(
+                    sp->libdeflate_dec, tif->tif_rawcp, (size_t)tif->tif_rawcc, op, (size_t)occ, NULL);
+
+                tif->tif_rawcp += tif->tif_rawcc;
+                tif->tif_rawcc = 0;
+
+                /* We accept LIBDEFLATE_INSUFFICIENT_SPACE has a return */
+                /* There are odd files in the wild where the last strip, when */
+                /* it is smaller in height than td_rowsperstrip, actually contains */
+                /* data for td_rowsperstrip lines. Just ignore that silently. */
+                if( res != LIBDEFLATE_SUCCESS &&
+                    res != LIBDEFLATE_INSUFFICIENT_SPACE )
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                                 "Decoding error at scanline %lu",
+                                 (unsigned long) tif->tif_row);
+                    return 0;
+                }
+
+                return 1;
+            }
+        } while(0);
+        sp->libdeflate_state = 0;
+#endif /* LIBDEFLATE_SUPPORT */
+
         sp->stream.next_in = tif->tif_rawcp;
         
 	sp->stream.next_out = op;
@@ -198,6 +276,7 @@ ZIPSetupEncode(TIFF* tif)
 {
 	static const char module[] = "ZIPSetupEncode";
 	ZIPState* sp = EncoderState(tif);
+        int cappedQuality;
 
 	assert(sp != NULL);
 	if (sp->state & ZSTATE_INIT_DECODE) {
@@ -205,7 +284,11 @@ ZIPSetupEncode(TIFF* tif)
 		sp->state = 0;
 	}
 
-	if (deflateInit(&sp->stream, sp->zipquality) != Z_OK) {
+        cappedQuality = sp->zipquality;
+        if( cappedQuality > Z_BEST_COMPRESSION )
+            cappedQuality = Z_BEST_COMPRESSION;
+
+	if (deflateInit(&sp->stream, cappedQuality) != Z_OK) {
 		TIFFErrorExt(tif->tif_clientdata, module, "%s", SAFE_MSG(sp));
 		return (0);
 	} else {
@@ -227,6 +310,9 @@ ZIPPreEncode(TIFF* tif, uint16 s)
 	if( sp->state != ZSTATE_INIT_ENCODE )
             tif->tif_setupencode( tif );
 
+#if LIBDEFLATE_SUPPORT
+        sp->libdeflate_state = -1;
+#endif
 	sp->stream.next_out = tif->tif_rawdata;
 	assert(sizeof(sp->stream.avail_out)==4);  /* if this assert gets raised,
 	    we need to simplify this code to reflect a ZLib that is likely updated
@@ -249,6 +335,95 @@ ZIPEncode(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s)
 	assert(sp->state == ZSTATE_INIT_ENCODE);
 
 	(void) s;
+
+#if LIBDEFLATE_SUPPORT
+        if( sp->libdeflate_state == 1 )
+            return 0;
+
+        /* If we have libdeflate support and we are asked to write a whole */
+        /* strip/tile, then go for using it */
+        do {
+            TIFFDirectory *td = &tif->tif_dir;
+
+            if( sp->libdeflate_state == 0 )
+                break;
+            if( sp->subcodec == DEFLATE_SUBCODEC_ZLIB )
+                break;
+
+            /* Libdeflate does not support the 0-compression level */
+            if( sp->zipquality == Z_NO_COMPRESSION )
+                break;
+
+            /* Check if we are in the situation where we can use libdeflate */
+            if (isTiled(tif)) {
+                if( TIFFTileSize64(tif) != (uint64)cc )
+                    break;
+            } else {
+                uint32 strip_height = td->td_imagelength - tif->tif_row;
+                if (strip_height > td->td_rowsperstrip)
+                    strip_height = td->td_rowsperstrip;
+                if( TIFFVStripSize64(tif, strip_height) != (uint64)cc )
+                    break;
+            }
+
+            /* Check for overflow */
+            if( (size_t)tif->tif_rawdatasize != (uint64)tif->tif_rawdatasize )
+                break;
+            if( (size_t)cc != (uint64)cc )
+                break;
+
+            /* Go for compression using libdeflate */
+            {
+                size_t nCompressedBytes;
+                if( sp->libdeflate_enc == NULL )
+                {
+                    /* To get results as good as zlib, we asked for an extra */
+                    /* level of compression */
+                    sp->libdeflate_enc = libdeflate_alloc_compressor(
+                        sp->zipquality == Z_DEFAULT_COMPRESSION ? 7 :
+                        sp->zipquality >= 6 && sp->zipquality <= 9 ? sp->zipquality + 1 :
+                        sp->zipquality);
+                    if( sp->libdeflate_enc == NULL )
+                    {
+                        TIFFErrorExt(tif->tif_clientdata, module,
+                                    "Cannot allocate compressor");
+                        break;
+                    }
+                }
+
+                /* Make sure the output buffer is large enough for the worse case. */
+                /* In TIFFWriteBufferSetup(), when libtiff allocates the buffer */
+                /* we've taken a 10% margin over the uncompressed size, which should */
+                /* be large enough even for the the worse case scenario. */
+                if( libdeflate_zlib_compress_bound(sp->libdeflate_enc, (size_t)cc) >
+                        (size_t)tif->tif_rawdatasize)
+                {
+                    break;
+                }
+
+                sp->libdeflate_state = 1;
+                nCompressedBytes = libdeflate_zlib_compress(
+                    sp->libdeflate_enc, bp, (size_t)cc, tif->tif_rawdata, (size_t)tif->tif_rawdatasize);
+
+                if( nCompressedBytes == 0 )
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                                 "Encoder error at scanline %lu",
+                                 (unsigned long) tif->tif_row);
+                    return 0;
+                }
+
+                tif->tif_rawcc = nCompressedBytes;
+
+                if( !TIFFFlushData1(tif) )
+                    return 0;
+
+                return 1;
+            }
+        } while(0);
+        sp->libdeflate_state = 0;
+#endif /* LIBDEFLATE_SUPPORT */
+
 	sp->stream.next_in = bp;
 	assert(sizeof(sp->stream.avail_in)==4);  /* if this assert gets raised,
 	    we need to simplify this code to reflect a ZLib that is likely updated
@@ -286,6 +461,11 @@ ZIPPostEncode(TIFF* tif)
 	ZIPState *sp = EncoderState(tif);
 	int state;
 
+#if LIBDEFLATE_SUPPORT
+        if( sp->libdeflate_state == 1 )
+            return 1;
+#endif
+
 	sp->stream.avail_in = 0;
 	do {
 		state = deflate(&sp->stream, Z_FINISH);
@@ -329,6 +509,14 @@ ZIPCleanup(TIFF* tif)
 		inflateEnd(&sp->stream);
 		sp->state = 0;
 	}
+
+#if LIBDEFLATE_SUPPORT
+        if( sp->libdeflate_dec )
+            libdeflate_free_decompressor(sp->libdeflate_dec);
+        if( sp->libdeflate_enc )
+            libdeflate_free_compressor(sp->libdeflate_enc);
+#endif
+
 	_TIFFfree(sp);
 	tif->tif_data = NULL;
 
@@ -344,15 +532,55 @@ ZIPVSetField(TIFF* tif, uint32 tag, va_list ap)
 	switch (tag) {
 	case TIFFTAG_ZIPQUALITY:
 		sp->zipquality = (int) va_arg(ap, int);
-		if ( sp->state&ZSTATE_INIT_ENCODE ) {
+                if( sp->zipquality < Z_DEFAULT_COMPRESSION ||
+                    sp->zipquality > LIBDEFLATE_MAX_COMPRESSION_LEVEL ) {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                                 "Invalid ZipQuality value. Should be in [-1,%d] range",
+                                 LIBDEFLATE_MAX_COMPRESSION_LEVEL);
+                    return 0;
+                }
+
+                if ( sp->state&ZSTATE_INIT_ENCODE ) {
+                        int cappedQuality = sp->zipquality;
+                        if( cappedQuality > Z_BEST_COMPRESSION )
+                            cappedQuality = Z_BEST_COMPRESSION;
 			if (deflateParams(&sp->stream,
-			    sp->zipquality, Z_DEFAULT_STRATEGY) != Z_OK) {
+			    cappedQuality, Z_DEFAULT_STRATEGY) != Z_OK) {
 				TIFFErrorExt(tif->tif_clientdata, module, "ZLib error: %s",
 					     SAFE_MSG(sp));
 				return (0);
 			}
 		}
+
+#if LIBDEFLATE_SUPPORT
+                if( sp->libdeflate_enc )
+                {
+                    libdeflate_free_compressor(sp->libdeflate_enc);
+                    sp->libdeflate_enc = NULL;
+                }
+#endif
+
 		return (1);
+
+        case TIFFTAG_DEFLATE_SUBCODEC:
+                sp->subcodec = (int) va_arg(ap, int);
+                if( sp->subcodec != DEFLATE_SUBCODEC_ZLIB &&
+                    sp->subcodec != DEFLATE_SUBCODEC_LIBDEFLATE )
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                                 "Invalid DeflateCodec value.");
+                    return 0;
+                }
+#if !LIBDEFLATE_SUPPORT
+                if( sp->subcodec == DEFLATE_SUBCODEC_LIBDEFLATE )
+                {
+                    TIFFErrorExt(tif->tif_clientdata, module,
+                                 "DeflateCodec = DEFLATE_SUBCODEC_LIBDEFLATE unsupported in this build");
+                    return 0;
+                }
+#endif
+                return 1;
+
 	default:
 		return (*sp->vsetparent)(tif, tag, ap);
 	}
@@ -368,6 +596,11 @@ ZIPVGetField(TIFF* tif, uint32 tag, va_list ap)
 	case TIFFTAG_ZIPQUALITY:
 		*va_arg(ap, int*) = sp->zipquality;
 		break;
+
+        case TIFFTAG_DEFLATE_SUBCODEC:
+		*va_arg(ap, int*) = sp->subcodec;
+		break;
+
 	default:
 		return (*sp->vgetparent)(tif, tag, ap);
 	}
@@ -376,6 +609,7 @@ ZIPVGetField(TIFF* tif, uint32 tag, va_list ap)
 
 static const TIFFField zipFields[] = {
     { TIFFTAG_ZIPQUALITY, 0, 0, TIFF_ANY, 0, TIFF_SETGET_INT, TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, TRUE, FALSE, "", NULL },
+    { TIFFTAG_DEFLATE_SUBCODEC, 0, 0, TIFF_ANY, 0, TIFF_SETGET_INT, TIFF_SETGET_UNDEFINED, FIELD_PSEUDO, TRUE, FALSE, "", NULL },
 };
 
 int
@@ -402,7 +636,7 @@ TIFFInitZIP(TIFF* tif, int scheme)
 	/*
 	 * Allocate state block so tag methods have storage to record values.
 	 */
-	tif->tif_data = (uint8*) _TIFFmalloc(sizeof (ZIPState));
+	tif->tif_data = (uint8*) _TIFFcalloc(sizeof (ZIPState), 1);
 	if (tif->tif_data == NULL)
 		goto bad;
 	sp = ZState(tif);
@@ -422,6 +656,11 @@ TIFFInitZIP(TIFF* tif, int scheme)
 	/* Default values for codec-specific fields */
 	sp->zipquality = Z_DEFAULT_COMPRESSION;	/* default comp. level */
 	sp->state = 0;
+#if LIBDEFLATE_SUPPORT
+        sp->subcodec = DEFLATE_SUBCODEC_LIBDEFLATE;
+#else
+        sp->subcodec = DEFLATE_SUBCODEC_ZLIB;
+#endif
 
 	/*
 	 * Install codec methods.
diff --git a/libtiff/tiff.h b/libtiff/tiff.h
index def9855d..eaf392e3 100644
--- a/libtiff/tiff.h
+++ b/libtiff/tiff.h
@@ -658,6 +658,9 @@ typedef enum {
 #define TIFFTAG_LERC_MAXZERROR          65567    /* LERC maximum error */
 #define TIFFTAG_WEBP_LEVEL		  65568	/* WebP compression level: WARNING not registered in Adobe-maintained registry */
 #define TIFFTAG_WEBP_LOSSLESS		65569	/* WebP lossless/lossy : WARNING not registered in Adobe-maintained registry */
+#define	TIFFTAG_DEFLATE_SUBCODEC	65570	/* ZIP codec: to get/set the sub-codec to use. Will default to libdeflate when available */
+#define     DEFLATE_SUBCODEC_ZLIB       0
+#define     DEFLATE_SUBCODEC_LIBDEFLATE 1
 
 /*
  * EXIF tags
diff --git a/libtiff/tiffconf.h.cmake.in b/libtiff/tiffconf.h.cmake.in
index 59542f1e..9b4b0328 100644
--- a/libtiff/tiffconf.h.cmake.in
+++ b/libtiff/tiffconf.h.cmake.in
@@ -87,6 +87,9 @@
 /* Support Deflate compression */
 #cmakedefine ZIP_SUPPORT 1
 
+/* Support libdeflate enhanced compression */
+#cmakedefine LIBDEFLATE_SUPPORT 1
+
 /* Support strip chopping (whether or not to convert single-strip uncompressed
    images to mutiple strips of ~8Kb to reduce memory usage) */
 #cmakedefine STRIPCHOP_DEFAULT 1
diff --git a/libtiff/tiffconf.h.in b/libtiff/tiffconf.h.in
index 5de30c9b..9bd6a2de 100644
--- a/libtiff/tiffconf.h.in
+++ b/libtiff/tiffconf.h.in
@@ -84,6 +84,9 @@
 /* Support Deflate compression */
 #undef ZIP_SUPPORT
 
+/* Support libdeflate enhanced compression */
+#undef LIBDEFLATE_SUPPORT
+
 /* Support strip chopping (whether or not to convert single-strip uncompressed
    images to mutiple strips of ~8Kb to reduce memory usage) */
 #undef STRIPCHOP_DEFAULT
diff --git a/man/tiffcp.1 b/man/tiffcp.1
index ec1875d2..6bfee348 100644
--- a/man/tiffcp.1
+++ b/man/tiffcp.1
@@ -151,6 +151,17 @@ e.g.
 for
 .SM Deflate
 encoding with maximum compression level and floating point predictor.
+.IP
+For the
+.SM Deflate
+codec, and in a libtiff build with libdeflate enabled, ``p12`` is
+actually the maximum level.
+.IP
+For the
+.SM Deflate
+codec, and in a libtiff build with libdeflate enabled, ``s0`` can be used to
+require zlib to be used, and ``s1`` for libdeflate (defaults to libdeflate when
+it is available).
 .TP
 .B \-f
 Specify the bit fill order to use in writing output data.
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 5a1a7420..bffb8bf7 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -109,7 +109,8 @@ set(TESTSCRIPTS
     tiff2rgba-ojpeg_zackthecat_subsamp22_single_strip.sh
     tiff2rgba-ojpeg_chewey_subsamp21_multi_strip.sh
     tiff2rgba-ojpeg_single_strip_no_rowsperstrip.sh
-    testfax4.sh)
+    testfax4.sh
+    testdeflatelaststripextradata.sh)
 
 # This list should contain all of the TIFF files in the 'images'
 # subdirectory which are intended to be used as input images for
diff --git a/test/Makefile.am b/test/Makefile.am
index 42e9f593..4973b614 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -150,6 +150,7 @@ TESTSCRIPTS = \
 	tiff2rgba-rgb-3c-16b.sh \
 	tiff2rgba-rgb-3c-8b.sh \
 	testfax4.sh \
+	testdeflatelaststripextradata.sh \
 	$(JPEG_DEPENDENT_TESTSCRIPTS)
 
 # This list should contain the references files
@@ -159,7 +160,8 @@ REFFILES = \
 	refs/o-tiff2ps-PS1.ps \
 	refs/o-tiff2ps-PS2.ps \
 	refs/o-tiff2ps-PS3.ps \
-	refs/o-testfax4.tiff
+	refs/o-testfax4.tiff \
+	refs/o-deflate-last-strip-extra-data.tiff
 
 # This list should contain all of the TIFF files in the 'images'
 # subdirectory which are intended to be used as input images for
@@ -181,7 +183,8 @@ TIFFIMAGES = \
 	images/ojpeg_zackthecat_subsamp22_single_strip.tiff \
 	images/ojpeg_chewey_subsamp21_multi_strip.tiff \
 	images/ojpeg_single_strip_no_rowsperstrip.tiff \
-	images/testfax4.tiff
+	images/testfax4.tiff \
+	images/deflate-last-strip-extra-data.tiff
 
 PNMIMAGES = \
 	images/minisblack-1c-8b.pgm \
diff --git a/test/images/README.txt b/test/images/README.txt
index b6447d8e..4a7ad829 100644
--- a/test/images/README.txt
+++ b/test/images/README.txt
@@ -30,3 +30,6 @@ PNM files:
 
 G3 Fax files :
   miniswhite-1c-1b.g3
+
+deflate-last-strip-extra-data.tiff is
+https://github.com/OSGeo/gdal/blob/master/autotest/alg/data/utmsmall_near.tiff
diff --git a/test/images/deflate-last-strip-extra-data.tiff b/test/images/deflate-last-strip-extra-data.tiff
new file mode 100644
index 00000000..e3ee9dcc
--- /dev/null
+++ b/test/images/deflate-last-strip-extra-data.tiff
diff --git a/test/refs/o-deflate-last-strip-extra-data.tiff b/test/refs/o-deflate-last-strip-extra-data.tiff
new file mode 100644
index 00000000..5d56fe54
--- /dev/null
+++ b/test/refs/o-deflate-last-strip-extra-data.tiff
diff --git a/test/testdeflatelaststripextradata.sh b/test/testdeflatelaststripextradata.sh
new file mode 100755
index 00000000..115eb226
--- /dev/null
+++ b/test/testdeflatelaststripextradata.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# check decoding of a deflate compressed file whose last strip which should
+# contain data for only 4 lines has more in it.
+. ${srcdir:-.}/common.sh
+infile="${IMAGES}/deflate-last-strip-extra-data.tiff"
+outfile="o-deflate-last-strip-extra-data.tiff"
+rm -f $outfile
+echo "$MEMCHECK ${TIFFCP} -c zip $infile $outfile"
+eval "$MEMCHECK ${TIFFCP} -c zip $infile $outfile"
+status=$?
+if [ $status != 0 ] ; then
+  echo "Returned failed status $status!"
+  echo "Output (if any) is in \"${outfile}\"."
+  exit $status
+fi
+echo "$MEMCHECK ${TIFFCMP} $outfile ${REFS}/$outfile"
+eval "$MEMCHECK ${TIFFCMP} $outfile ${REFS}/$outfile"
+status=$?
+if [ $status != 0 ] ; then
+  echo "Returned failed status $status!"
+  echo "\"${outfile}\" differs from reference file."
+  exit $status
+fi
+
+outfile="o-deflate-last-strip-extra-data-tiled.tiff"
+rm -f $outfile
+echo "$MEMCHECK ${TIFFCP} -c zip -t -w 256 -l 256 $infile $outfile"
+eval "$MEMCHECK ${TIFFCP} -c zip -t -w 256 -l 256 $infile $outfile"
+status=$?
+if [ $status != 0 ] ; then
+  echo "Returned failed status $status!"
+  echo "Output (if any) is in \"${outfile}\"."
+  exit $status
+fi
+echo "$MEMCHECK ${TIFFCMP} $outfile ${REFS}/o-deflate-last-strip-extra-data.tiff"
+eval "$MEMCHECK ${TIFFCMP} $outfile ${REFS}/o-deflate-last-strip-extra-data.tiff"
+status=$?
+if [ $status != 0 ] ; then
+  echo "Returned failed status $status!"
+  echo "\"${outfile}\" differs from reference file."
+  exit $status
+fi
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index 9dbcf67a..e56b1c10 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -97,6 +97,7 @@ static int jpegcolormode = JPEGCOLORMODE_RGB;
 static uint16 defcompression = (uint16) -1;
 static uint16 defpredictor = (uint16) -1;
 static int defpreset =  -1;
+static int subcodec = -1;
 
 static int tiffcp(TIFF*, TIFF*);
 static int processCompressOptions(char*);
@@ -361,6 +362,8 @@ processZIPOptions(char* cp)
 				defpredictor = atoi(cp);
 			else if (*cp == 'p')
 				defpreset = atoi(++cp);
+			else if (*cp == 's')
+				subcodec = atoi(++cp);
 			else
 				usage(EXIT_FAILURE);
 		} while( (cp = strchr(cp, ':')) );
@@ -494,6 +497,9 @@ static const char* stuff[] = {
 "LZW, Deflate (ZIP), LZMA2, ZSTD and WEBP options:",
 " #               set predictor value",
 " p#              set compression level (preset)",
+#if LIBDEFLATE_SUPPORT
+" s#              set subcodec (0=zlib, 1=libdeflate) (only for Deflate/ZIP)",
+#endif
 "For example, -c lzw:2 to get LZW-encoded data with horizontal differencing,",
 "-c zip:3:p9 for Deflate encoding with maximum compression level and floating",
 "point predictor.",
@@ -780,6 +786,17 @@ tiffcp(TIFF* in, TIFF* out)
 				TIFFSetField(out, TIFFTAG_PREDICTOR, predictor);
 			else
 				CopyField(TIFFTAG_PREDICTOR, predictor);
+                        if( compression == COMPRESSION_ADOBE_DEFLATE ||
+                            compression == COMPRESSION_DEFLATE )
+                        {
+                            if( subcodec != -1 )
+                            {
+                                if( TIFFSetField(out, TIFFTAG_DEFLATE_SUBCODEC, subcodec) != 1 )
+                                {
+                                    return FALSE;
+                                }
+                            }
+                        }
 			/*fallthrough*/
 		case COMPRESSION_WEBP:
 			if (preset != -1) {