diff options
author | Keith Bostic <keith.bostic@mongodb.com> | 2016-10-21 23:10:39 -0400 |
---|---|---|
committer | Michael Cahill <michael.cahill@mongodb.com> | 2016-10-21 23:10:39 -0400 |
commit | b486c5ce2f342f4ec659b31764320d740ab40f46 (patch) | |
tree | f711932b7d3e869695d3e400e7ba4718e2b87cb9 /build_posix | |
parent | ceeb57b565fca6ade4bb02d8cf62095374743bd1 (diff) | |
download | mongo-b486c5ce2f342f4ec659b31764320d740ab40f46.tar.gz |
WT-2880 Add Zstandard compression support (#3075)
* Fix a bug found by inspection in LZ4 code: we're going to offset the destination buffer by sizeof(LZ4_PREFIX), so we need to offset the destination buffer length by the same amount.
* Prettiness pass through the snappy driver so it and the zstd driver look the same, minor cleanup in zlib.
* Add the compression_level configuration option to the zstd extension initialization function so it's possible to set the compression level from an application.
* Fix a bug in zlib raw compression: the destination buffer size (dst_len), can be smaller than the target page size (page_max), use the smaller of the two values to set the target compression size.
* The zlib raw compression function could return without calling deflateEnd on its z_stream structures, potentially leaking memory and scratch buffers.
* If the default reserved bytes for zlib raw compression isn't sufficient, we fail on compression of what might be very large blocks. We don't have information on how many bytes need to be reserved in order to know the final deflate() will succeed, and the default value was experimentally derived, for all we know there are workloads where the default will fail
a lot. Add a fallback before failing hard, try with 2x the default reserved space.
Diffstat (limited to 'build_posix')
-rw-r--r-- | build_posix/Make.base | 3 | ||||
-rw-r--r-- | build_posix/Make.subdirs | 1 | ||||
-rw-r--r-- | build_posix/aclocal/options.m4 | 34 |
3 files changed, 37 insertions, 1 deletions
diff --git a/build_posix/Make.base b/build_posix/Make.base index 4efbe3f76c3..5b945aca5e0 100644 --- a/build_posix/Make.base +++ b/build_posix/Make.base @@ -77,6 +77,9 @@ endif if HAVE_BUILTIN_EXTENSION_ZLIB libwiredtiger_la_LIBADD += ext/compressors/zlib/libwiredtiger_zlib.la endif +if HAVE_BUILTIN_EXTENSION_ZSTD +libwiredtiger_la_LIBADD += ext/compressors/zstd/libwiredtiger_zstd.la +endif libwiredtiger_static_la_LIBADD=$(libwiredtiger_la_LIBADD) libwiredtiger_static_la_SOURCES=$(libwiredtiger_la_SOURCES) diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs index 0b5175e4196..55941837249 100644 --- a/build_posix/Make.subdirs +++ b/build_posix/Make.subdirs @@ -11,6 +11,7 @@ ext/compressors/lz4 LZ4 ext/compressors/nop ext/compressors/snappy SNAPPY ext/compressors/zlib ZLIB +ext/compressors/zstd ZSTD ext/datasources/helium HAVE_HELIUM ext/encryptors/nop ext/encryptors/rotn diff --git a/build_posix/aclocal/options.m4 b/build_posix/aclocal/options.m4 index 1f6a1690279..7043430a6d6 100644 --- a/build_posix/aclocal/options.m4 +++ b/build_posix/aclocal/options.m4 @@ -19,10 +19,12 @@ AH_TEMPLATE(HAVE_BUILTIN_EXTENSION_SNAPPY, [Snappy support automatically loaded.]) AH_TEMPLATE(HAVE_BUILTIN_EXTENSION_ZLIB, [Zlib support automatically loaded.]) +AH_TEMPLATE(HAVE_BUILTIN_EXTENSION_ZSTD, + [ZSTD support automatically loaded.]) AC_MSG_CHECKING(if --with-builtins option specified) AC_ARG_WITH(builtins, [AS_HELP_STRING([--with-builtins], - [builtin extension names (lz4, snappy, zlib).])], + [builtin extension names (lz4, snappy, zlib, zstd).])], [with_builtins=$withval], [with_builtins=]) @@ -36,6 +38,8 @@ for builtin_i in $builtin_list; do wt_cv_with_builtin_extension_snappy=yes;; zlib) AC_DEFINE(HAVE_BUILTIN_EXTENSION_ZLIB) wt_cv_with_builtin_extension_zlib=yes;; + zstd) AC_DEFINE(HAVE_BUILTIN_EXTENSION_ZSTD) + wt_cv_with_builtin_extension_zstd=yes;; *) AC_MSG_ERROR([Unknown builtin extension "$builtin_i"]);; esac done @@ -45,6 +49,8 @@ AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_SNAPPY], [test "$wt_cv_with_builtin_extension_snappy" = "yes"]) AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_ZLIB], [test "$wt_cv_with_builtin_extension_zlib" = "yes"]) +AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_ZSTD], + [test "$wt_cv_with_builtin_extension_zstd" = "yes"]) AC_MSG_RESULT($with_builtins) AH_TEMPLATE( @@ -276,4 +282,30 @@ if test "$wt_cv_enable_zlib" = "yes"; then fi AM_CONDITIONAL([ZLIB], [test "$wt_cv_enable_zlib" = "yes"]) +AC_MSG_CHECKING(if --enable-zstd option specified) +AC_ARG_ENABLE(zstd, + [AS_HELP_STRING([--enable-zstd], + [Build the zstd compressor extension.])], r=$enableval, r=no) +case "$r" in +no) if test "$wt_cv_with_builtin_extension_zstd" = "yes"; then + wt_cv_enable_zstd=yes + else + wt_cv_enable_zstd=no + fi + ;; +*) if test "$wt_cv_with_builtin_extension_zstd" = "yes"; then + AC_MSG_ERROR( + [Only one of --enable-zstd --with-builtins=zstd allowed]) + fi + wt_cv_enable_zstd=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_zstd) +if test "$wt_cv_enable_zstd" = "yes"; then + AC_CHECK_HEADER(zstd.h,, + [AC_MSG_ERROR([--enable-zstd requires zstd.h])]) + AC_CHECK_LIB(zstd, ZSTD_compress,, + [AC_MSG_ERROR([--enable-zstd requires Zstd library])]) +fi +AM_CONDITIONAL([ZSTD], [test "$wt_cv_enable_zstd" = "yes"]) + ]) |