From b486c5ce2f342f4ec659b31764320d740ab40f46 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 21 Oct 2016 23:10:39 -0400 Subject: WT-2880 Add Zstandard compression support (#3075) * Fix a bug found by inspection in LZ4 code: we're going to offset the destination buffer by sizeof(LZ4_PREFIX), so we need to offset the destination buffer length by the same amount. * Prettiness pass through the snappy driver so it and the zstd driver look the same, minor cleanup in zlib. * Add the compression_level configuration option to the zstd extension initialization function so it's possible to set the compression level from an application. * Fix a bug in zlib raw compression: the destination buffer size (dst_len), can be smaller than the target page size (page_max), use the smaller of the two values to set the target compression size. * The zlib raw compression function could return without calling deflateEnd on its z_stream structures, potentially leaking memory and scratch buffers. * If the default reserved bytes for zlib raw compression isn't sufficient, we fail on compression of what might be very large blocks. We don't have information on how many bytes need to be reserved in order to know the final deflate() will succeed, and the default value was experimentally derived, for all we know there are workloads where the default will fail a lot. Add a fallback before failing hard, try with 2x the default reserved space. --- bench/wtperf/wtperf.c | 5 +++++ bench/wtperf/wtperf.h | 3 +++ bench/wtperf/wtperf_opt.i | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) (limited to 'bench') diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index a1dc78a5312..8c7f0053388 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -2078,6 +2078,11 @@ config_compress(WTPERF *wtperf) wtperf->compress_ext = ZLIB_EXT; #endif wtperf->compress_table = ZLIB_BLK; + } else if (strcmp(s, "zstd") == 0) { +#ifndef HAVE_BUILTIN_EXTENSION_ZSTD + wtperf->compress_ext = ZSTD_EXT; +#endif + wtperf->compress_table = ZSTD_BLK; } else { fprintf(stderr, "invalid compression configuration: %s\n", s); diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index afce017d919..81d74e134f6 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -54,6 +54,9 @@ typedef struct __truncate_queue_entry TRUNCATE_QUEUE_ENTRY; #define ZLIB_BLK BLKCMP_PFX "zlib" #define ZLIB_EXT \ EXT_PFX EXTPATH "zlib/.libs/libwiredtiger_zlib.so" EXT_SFX +#define ZSTD_BLK BLKCMP_PFX "zstd" +#define ZSTD_EXT \ + EXT_PFX EXTPATH "zstd/.libs/libwiredtiger_zstd.so" EXT_SFX typedef struct { int64_t threads; /* Thread count */ diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i index d2d255b38c2..680eb53a90e 100644 --- a/bench/wtperf/wtperf_opt.i +++ b/bench/wtperf/wtperf_opt.i @@ -100,7 +100,7 @@ DEF_OPT_AS_BOOL(close_conn, 1, "properly close connection at end of test. " DEF_OPT_AS_BOOL(compact, 0, "post-populate compact for LSM merging activity") DEF_OPT_AS_STRING(compression, "none", "compression extension. Allowed configuration values are: " - "'none', 'lz4', 'snappy', 'zlib'") + "'none', 'lz4', 'snappy', 'zlib', 'zstd'") DEF_OPT_AS_BOOL(create, 1, "do population phase; false to use existing database") DEF_OPT_AS_UINT32(database_count, 1, -- cgit v1.2.1