diff options
author | Michael Cahill <michael.cahill@mongodb.com> | 2016-01-29 12:36:27 +1100 |
---|---|---|
committer | Michael Cahill <michael.cahill@mongodb.com> | 2016-01-29 12:36:27 +1100 |
commit | 7bd982093b2a14760951177af47ee21e84ce16a4 (patch) | |
tree | e510da3b96cecd7b04a9ff549e396a0246f1c922 | |
parent | e3d59fb590713c390ca21d414eff0575877ef5e6 (diff) | |
parent | dea25e13b8e15a8528f374656ce2e8432a66c9f4 (diff) | |
download | mongo-7bd982093b2a14760951177af47ee21e84ce16a4.tar.gz |
Merge branch 'develop' into WT-60
-rw-r--r-- | bench/wtperf/config.c | 3 | ||||
-rw-r--r-- | bench/wtperf/wtperf.h | 3 | ||||
-rw-r--r-- | bench/wtperf/wtperf_opt.i | 2 | ||||
-rw-r--r-- | build_posix/Make.subdirs | 1 | ||||
-rw-r--r-- | build_posix/aclocal/options.m4 | 17 | ||||
-rw-r--r-- | build_win/wiredtiger_config.h | 3 | ||||
-rw-r--r-- | dist/api_data.py | 9 | ||||
-rw-r--r-- | dist/extlist | 1 | ||||
-rw-r--r-- | examples/c/ex_all.c | 15 | ||||
-rw-r--r-- | examples/java/com/wiredtiger/examples/ex_all.java | 13 | ||||
-rw-r--r-- | ext/compressors/bzip2/Makefile.am | 6 | ||||
-rw-r--r-- | ext/compressors/bzip2/bzip2_compress.c | 415 | ||||
-rw-r--r-- | src/docs/build-posix.dox | 4 | ||||
-rw-r--r-- | src/docs/compression.dox | 31 | ||||
-rw-r--r-- | src/docs/upgrading.dox | 7 | ||||
-rw-r--r-- | src/docs/wtperf.dox | 2 | ||||
-rw-r--r-- | src/include/wiredtiger.in | 22 | ||||
-rw-r--r-- | test/format/config.c | 52 | ||||
-rw-r--r-- | test/format/config.h | 3 | ||||
-rw-r--r-- | test/format/format.h | 16 | ||||
-rw-r--r-- | test/format/wts.c | 7 | ||||
-rw-r--r-- | test/suite/test_compress01.py | 1 | ||||
-rw-r--r-- | test/suite/test_cursor08.py | 1 | ||||
-rw-r--r-- | test/suite/test_encrypt01.py | 2 | ||||
-rw-r--r-- | test/suite/test_txn07.py | 1 |
25 files changed, 60 insertions, 577 deletions
diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index d9a22f4708d..3cb20ff2b26 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -156,9 +156,6 @@ config_compress(CONFIG *cfg) if (strcmp(s, "none") == 0) { cfg->compress_ext = NULL; cfg->compress_table = NULL; - } else if (strcmp(s, "bzip") == 0) { - cfg->compress_ext = BZIP_EXT; - cfg->compress_table = BZIP_BLK; } else if (strcmp(s, "lz4") == 0) { cfg->compress_ext = LZ4_EXT; cfg->compress_table = LZ4_BLK; diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 7dbe1822a26..f8d5a5c0bc9 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -73,9 +73,6 @@ typedef struct __truncate_queue_entry TRUNCATE_QUEUE_ENTRY; #define EXTPATH "../../ext/compressors/" /* Extensions path */ #define BLKCMP_PFX ",block_compressor=" -#define BZIP_BLK BLKCMP_PFX "bzip2" -#define BZIP_EXT \ - EXT_PFX EXTPATH "bzip2/.libs/libwiredtiger_bzip2.so" EXT_SFX #define LZ4_BLK BLKCMP_PFX "lz4" #define LZ4_EXT \ EXT_PFX EXTPATH "lz4/.libs/libwiredtiger_lz4.so" EXT_SFX diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i index 6dc2a6d5569..60bbaff56e5 100644 --- a/bench/wtperf/wtperf_opt.i +++ b/bench/wtperf/wtperf_opt.i @@ -97,7 +97,7 @@ DEF_OPT_AS_CONFIG_STRING(conn_config, "create", DEF_OPT_AS_BOOL(compact, 0, "post-populate compact for LSM merging activity") DEF_OPT_AS_STRING(compression, "none", "compression extension. Allowed configuration values are: " - "'none', 'bzip', 'lz4', 'snappy', 'zlib'") + "'none', 'lz4', 'snappy', 'zlib'") DEF_OPT_AS_BOOL(create, 1, "do population phase; false to use existing database") DEF_OPT_AS_UINT32(database_count, 1, diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs index e2f128a48df..e1f8a05c613 100644 --- a/build_posix/Make.subdirs +++ b/build_posix/Make.subdirs @@ -6,7 +6,6 @@ # If the directory exists, it is added to AUTO_SUBDIRS. # If a condition is included, the subdir is made conditional via AM_CONDITIONAL ext/collators/reverse -ext/compressors/bzip2 BZIP2 ext/compressors/lz4 LZ4 ext/compressors/nop ext/compressors/snappy SNAPPY diff --git a/build_posix/aclocal/options.m4 b/build_posix/aclocal/options.m4 index 01d08ce3d16..9422c65822d 100644 --- a/build_posix/aclocal/options.m4 +++ b/build_posix/aclocal/options.m4 @@ -47,23 +47,6 @@ AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_ZLIB], [test "$wt_cv_with_builtin_extension_zlib" = "yes"]) AC_MSG_RESULT($with_builtins) -AC_MSG_CHECKING(if --enable-bzip2 option specified) -AC_ARG_ENABLE(bzip2, - [AS_HELP_STRING([--enable-bzip2], - [Build the bzip2 compressor extension.])], r=$enableval, r=no) -case "$r" in -no) wt_cv_enable_bzip2=no;; -*) wt_cv_enable_bzip2=yes;; -esac -AC_MSG_RESULT($wt_cv_enable_bzip2) -if test "$wt_cv_enable_bzip2" = "yes"; then - AC_CHECK_HEADER(bzlib.h,, - [AC_MSG_ERROR([--enable-bzip2 requires bzlib.h])]) - AC_CHECK_LIB(bz2, BZ2_bzCompress,, - [AC_MSG_ERROR([--enable-bzip2 requires bz2 library])]) -fi -AM_CONDITIONAL([BZIP2], [test "$wt_cv_enable_bzip2" = "yes"]) - AH_TEMPLATE(HAVE_DIAGNOSTIC, [Define to 1 for diagnostic tests.]) AC_MSG_CHECKING(if --enable-diagnostic option specified) AC_ARG_ENABLE(diagnostic, diff --git a/build_win/wiredtiger_config.h b/build_win/wiredtiger_config.h index d1ed307cb85..33dbc9b724d 100644 --- a/build_win/wiredtiger_config.h +++ b/build_win/wiredtiger_config.h @@ -52,9 +52,6 @@ /* Define to 1 if you have the <inttypes.h> header file. */ #define HAVE_INTTYPES_H 1 -/* Define to 1 if you have the `bz2' library (-lbz2). */ -/* #undef HAVE_LIBBZ2 */ - /* Define to 1 if you have the `dl' library (-ldl). */ /* #undef HAVE_LIBDL */ diff --git a/dist/api_data.py b/dist/api_data.py index e4055112d5e..c386c0b345d 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -136,8 +136,8 @@ file_config = format_meta + [ configure a compressor for file blocks. Permitted values are \c "none" or custom compression engine name created with WT_CONNECTION::add_compressor. If WiredTiger has builtin support for - \c "bzip2", \c "snappy", \c "lz4" or \c "zlib" compression, these names - are also available. See @ref compression for more information'''), + \c "snappy", \c "lz4" or \c "zlib" compression, these names are also + available. See @ref compression for more information'''), Config('cache_resident', 'false', r''' do not ever evict the object's pages from cache. Not compatible with LSM tables; see @ref tuning_cache_resident for more information''', @@ -422,9 +422,8 @@ connection_runtime_config = [ configure a compressor for log records. Permitted values are \c "none" or custom compression engine name created with WT_CONNECTION::add_compressor. If WiredTiger has builtin support - for \c "bzip2", \c "snappy", \c "lz4" or \c "zlib" compression, - these names are also available. See @ref compression for more - information'''), + for \c "snappy", \c "lz4" or \c "zlib" compression, these names + are also available. See @ref compression for more information'''), Config('enabled', 'false', r''' enable logging subsystem''', type='boolean'), diff --git a/dist/extlist b/dist/extlist index 874d21289d2..a5515642d48 100644 --- a/dist/extlist +++ b/dist/extlist @@ -2,7 +2,6 @@ # List of extension source files for WiredTiger library. ext/collators/reverse/reverse_collator.c -ext/compressors/bzip2/bzip2_compress.c ext/compressors/nop/nop_compress.c ext/compressors/snappy/snappy_compress.c ext/compressors/zlib/zlib_compress.c diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index f7ee857a7c7..418c99ad6a3 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -591,13 +591,6 @@ session_ops(WT_SESSION *session) * the code snippets, use #ifdef's to avoid running it. */ #ifdef MIGHT_NOT_RUN - /*! [Create a bzip2 compressed table] */ - ret = session->create(session, - "table:mytable", - "block_compressor=bzip2,key_format=S,value_format=S"); - /*! [Create a bzip2 compressed table] */ - ret = session->drop(session, "table:mytable", NULL); - /*! [Create a lz4 compressed table] */ ret = session->create(session, "table:mytable", @@ -1084,14 +1077,6 @@ main(void) * be installed, causing the open to fail. The documentation requires * the code snippets, use #ifdef's to avoid running it. */ - /*! [Configure bzip2 extension] */ - ret = wiredtiger_open(home, NULL, - "create," - "extensions=[/usr/local/lib/libwiredtiger_bzip2.so]", &conn); - /*! [Configure bzip2 extension] */ - if (ret == 0) - (void)conn->close(conn, NULL); - /*! [Configure lz4 extension] */ ret = wiredtiger_open(home, NULL, "create," diff --git a/examples/java/com/wiredtiger/examples/ex_all.java b/examples/java/com/wiredtiger/examples/ex_all.java index 153f12d3e27..09db8e0fd56 100644 --- a/examples/java/com/wiredtiger/examples/ex_all.java +++ b/examples/java/com/wiredtiger/examples/ex_all.java @@ -516,12 +516,6 @@ session_ops(Session session) * the code snippets, use if (false) to avoid running it. */ if (false) { // MIGHT_NOT_RUN - /*! [Create a bzip2 compressed table] */ - ret = session.create("table:mytable", - "block_compressor=bzip2,key_format=S,value_format=S"); - /*! [Create a bzip2 compressed table] */ - ret = session.drop("table:mytable", null); - /*! [Create a lz4 compressed table] */ ret = session.create("table:mytable", "block_compressor=lz4,key_format=S,value_format=S"); @@ -899,13 +893,6 @@ allExample() * be installed, causing the open to fail. The documentation requires * the code snippets, use if (false) to avoid running it. */ - /*! [Configure bzip2 extension] */ - conn = wiredtiger.open(home, - "create," + - "extensions=[/usr/local/lib/libwiredtiger_bzip2.so]"); - /*! [Configure bzip2 extension] */ - conn.close(null); - /*! [Configure lz4 extension] */ conn = wiredtiger.open(home, "create," + diff --git a/ext/compressors/bzip2/Makefile.am b/ext/compressors/bzip2/Makefile.am deleted file mode 100644 index 0aedc2efd80..00000000000 --- a/ext/compressors/bzip2/Makefile.am +++ /dev/null @@ -1,6 +0,0 @@ -AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include - -lib_LTLIBRARIES = libwiredtiger_bzip2.la -libwiredtiger_bzip2_la_SOURCES = bzip2_compress.c -libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module -libwiredtiger_bzip2_la_LIBADD = -lbz2 diff --git a/ext/compressors/bzip2/bzip2_compress.c b/ext/compressors/bzip2/bzip2_compress.c deleted file mode 100644 index 845107c609f..00000000000 --- a/ext/compressors/bzip2/bzip2_compress.c +++ /dev/null @@ -1,415 +0,0 @@ -/*- - * Public Domain 2014-2016 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include <bzlib.h> -#include <errno.h> -#include <inttypes.h> -#include <stdlib.h> -#include <string.h> - -#include <wiredtiger.h> -#include <wiredtiger_ext.h> - -/* Local compressor structure. */ -typedef struct { - WT_COMPRESSOR compressor; /* Must come first */ - - WT_EXTENSION_API *wt_api; /* Extension API */ - - int bz_verbosity; /* Configuration */ - int bz_blocksize100k; - int bz_workfactor; - int bz_small; -} BZIP_COMPRESSOR; - -/* - * Bzip gives us a cookie to pass to the underlying allocation functions; we - * we need two handles, package them up. - */ -typedef struct { - WT_COMPRESSOR *compressor; - WT_SESSION *session; -} BZIP_OPAQUE; - -/* - * bzip2_error -- - * Output an error message, and return a standard error code. - */ -static int -bzip2_error( - WT_COMPRESSOR *compressor, WT_SESSION *session, const char *call, int bzret) -{ - WT_EXTENSION_API *wt_api; - const char *msg; - - wt_api = ((BZIP_COMPRESSOR *)compressor)->wt_api; - - switch (bzret) { - case BZ_MEM_ERROR: - msg = "BZ_MEM_ERROR"; - break; - case BZ_OUTBUFF_FULL: - msg = "BZ_OUTBUFF_FULL"; - break; - case BZ_SEQUENCE_ERROR: - msg = "BZ_SEQUENCE_ERROR"; - break; - case BZ_PARAM_ERROR: - msg = "BZ_PARAM_ERROR"; - break; - case BZ_DATA_ERROR: - msg = "BZ_DATA_ERROR"; - break; - case BZ_DATA_ERROR_MAGIC: - msg = "BZ_DATA_ERROR_MAGIC"; - break; - case BZ_IO_ERROR: - msg = "BZ_IO_ERROR"; - break; - case BZ_UNEXPECTED_EOF: - msg = "BZ_UNEXPECTED_EOF"; - break; - case BZ_CONFIG_ERROR: - msg = "BZ_CONFIG_ERROR"; - break; - default: - msg = "unknown error"; - break; - } - - (void)wt_api->err_printf(wt_api, session, - "bzip2 error: %s: %s: %d", call, msg, bzret); - return (WT_ERROR); -} - -/* - * bzalloc -- - * Allocate scratch buffers. - */ -static void * -bzalloc(void *cookie, int number, int size) -{ - BZIP_OPAQUE *opaque; - WT_EXTENSION_API *wt_api; - - opaque = cookie; - wt_api = ((BZIP_COMPRESSOR *)opaque->compressor)->wt_api; - return (wt_api->scr_alloc( - wt_api, opaque->session, (size_t)(number * size))); -} - -/* - * bzfree -- - * Free scratch buffers. - */ -static void -bzfree(void *cookie, void *p) -{ - BZIP_OPAQUE *opaque; - WT_EXTENSION_API *wt_api; - - opaque = cookie; - wt_api = ((BZIP_COMPRESSOR *)opaque->compressor)->wt_api; - wt_api->scr_free(wt_api, opaque->session, p); -} - -/* - * bzip2_compress -- - * WiredTiger bzip2 compression. - */ -static int -bzip2_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, - uint8_t *src, size_t src_len, - uint8_t *dst, size_t dst_len, - size_t *result_lenp, int *compression_failed) -{ - BZIP_COMPRESSOR *bzip_compressor; - BZIP_OPAQUE opaque; - bz_stream bz; - int ret; - - bzip_compressor = (BZIP_COMPRESSOR *)compressor; - - memset(&bz, 0, sizeof(bz)); - bz.bzalloc = bzalloc; - bz.bzfree = bzfree; - opaque.compressor = compressor; - opaque.session = session; - bz.opaque = &opaque; - - if ((ret = BZ2_bzCompressInit(&bz, - bzip_compressor->bz_blocksize100k, - bzip_compressor->bz_verbosity, - bzip_compressor->bz_workfactor)) != BZ_OK) - return (bzip2_error( - compressor, session, "BZ2_bzCompressInit", ret)); - - bz.next_in = (char *)src; - bz.avail_in = (uint32_t)src_len; - bz.next_out = (char *)dst; - bz.avail_out = (uint32_t)dst_len; - if ((ret = BZ2_bzCompress(&bz, BZ_FINISH)) == BZ_STREAM_END) { - *compression_failed = 0; - *result_lenp = dst_len - bz.avail_out; - } else - *compression_failed = 1; - - if ((ret = BZ2_bzCompressEnd(&bz)) != BZ_OK) - return ( - bzip2_error(compressor, session, "BZ2_bzCompressEnd", ret)); - - return (0); -} - -/* - * __bzip2_compress_raw_random -- - * Return a 32-bit pseudo-random number. - * - * This is an implementation of George Marsaglia's multiply-with-carry pseudo- - * random number generator. Computationally fast, with reasonable randomness - * properties. - */ -static uint32_t -__bzip2_compress_raw_random(void) -{ - static uint32_t m_w = 521288629; - static uint32_t m_z = 362436069; - - m_z = 36969 * (m_z & 65535) + (m_z >> 16); - m_w = 18000 * (m_w & 65535) + (m_w >> 16); - return (m_z << 16) + (m_w & 65535); -} - -/* - * bzip2_compress_raw -- - * Test function for the test/format utility. - */ -static int -bzip2_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, - size_t page_max, int split_pct, size_t extra, - uint8_t *src, uint32_t *offsets, uint32_t slots, - uint8_t *dst, size_t dst_len, int final, - size_t *result_lenp, uint32_t *result_slotsp) -{ - uint32_t take, twenty_pct; - int compression_failed, ret; - - (void)page_max; /* Unused parameters */ - (void)split_pct; - (void)extra; - (void)final; - - /* - * This function is used by the test/format utility to test the - * WT_COMPRESSOR::compress_raw functionality. - * - * I'm trying to mimic how a real application is likely to behave: if - * it's a small number of slots, we're not going to take them because - * they aren't worth compressing. In all likelihood, that's going to - * be because the btree is wrapping up a page, but that's OK, that is - * going to happen a lot. In addition, add a 2% chance of not taking - * anything at all just because we don't want to take it. Otherwise, - * select between 80 and 100% of the slots and compress them, stepping - * down by 5 slots at a time until something works. - */ - take = slots; - if (take < 10 || __bzip2_compress_raw_random() % 100 < 2) - take = 0; - else { - twenty_pct = (slots / 10) * 2; - if (twenty_pct < slots) - take -= __bzip2_compress_raw_random() % twenty_pct; - - for (;;) { - if ((ret = bzip2_compress(compressor, session, - src, offsets[take], - dst, dst_len, - result_lenp, &compression_failed)) != 0) - return (ret); - if (!compression_failed) - break; - if (take < 10) { - take = 0; - break; - } - take -= 5; - } - } - - *result_slotsp = take; - if (take == 0) - *result_lenp = 0; - -#if 0 - fprintf(stderr, - "bzip2_compress_raw (%s): page_max %" PRIuMAX - ", split_pct %u, extra %" PRIuMAX - ", slots %" PRIu32 ", take %" PRIu32 ": %" PRIu32 " -> %" - PRIuMAX "\n", - final ? "final" : "not final", - (uintmax_t)page_max, split_pct, (uintmax_t)extra, - slots, take, offsets[take], (uintmax_t)*result_lenp); -#endif - return (take == 0 ? EAGAIN : 0); -} - -/* - * bzip2_decompress -- - * WiredTiger bzip2 decompression. - */ -static int -bzip2_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, - uint8_t *src, size_t src_len, - uint8_t *dst, size_t dst_len, - size_t *result_lenp) -{ - BZIP_COMPRESSOR *bzip_compressor; - BZIP_OPAQUE opaque; - bz_stream bz; - int ret, tret; - - bzip_compressor = (BZIP_COMPRESSOR *)compressor; - - memset(&bz, 0, sizeof(bz)); - bz.bzalloc = bzalloc; - bz.bzfree = bzfree; - opaque.compressor = compressor; - opaque.session = session; - bz.opaque = &opaque; - - if ((ret = BZ2_bzDecompressInit(&bz, - bzip_compressor->bz_small, bzip_compressor->bz_verbosity)) != BZ_OK) - return (bzip2_error( - compressor, session, "BZ2_bzDecompressInit", ret)); - - bz.next_in = (char *)src; - bz.avail_in = (uint32_t)src_len; - bz.next_out = (char *)dst; - bz.avail_out = (uint32_t)dst_len; - if ((ret = BZ2_bzDecompress(&bz)) == BZ_STREAM_END) { - *result_lenp = dst_len - bz.avail_out; - ret = 0; - } else { - /* - * If BZ2_bzDecompress returns 0, it expects there to be more - * data available. There isn't, so treat this as an error. - */ - if (ret == 0) - ret = BZ_DATA_ERROR; - (void)bzip2_error(compressor, session, "BZ2_bzDecompress", ret); - } - - if ((tret = BZ2_bzDecompressEnd(&bz)) != BZ_OK) - return (bzip2_error( - compressor, session, "BZ2_bzDecompressEnd", tret)); - - return (ret == 0 ? - 0 : bzip2_error(compressor, session, "BZ2_bzDecompressEnd", ret)); -} - -/* - * bzip2_terminate -- - * WiredTiger bzip2 compression termination. - */ -static int -bzip2_terminate(WT_COMPRESSOR *compressor, WT_SESSION *session) -{ - (void)session; /* Unused parameters */ - - free(compressor); - return (0); -} - -/* - * bzip2_add_compressor -- - * Add a bzip2 compressor. - */ -static int -bzip2_add_compressor(WT_CONNECTION *connection, int raw, const char *name) -{ - BZIP_COMPRESSOR *bzip_compressor; - - /* - * There are two almost identical bzip2 compressors: one supporting raw - * compression (used by test/format to test raw compression), the other - * without raw compression, that might be useful for real applications. - */ - if ((bzip_compressor = calloc(1, sizeof(BZIP_COMPRESSOR))) == NULL) - return (errno); - - bzip_compressor->compressor.compress = bzip2_compress; - bzip_compressor-> - compressor.compress_raw = raw ? bzip2_compress_raw : NULL; - bzip_compressor->compressor.decompress = bzip2_decompress; - bzip_compressor->compressor.pre_size = NULL; - bzip_compressor->compressor.terminate = bzip2_terminate; - - bzip_compressor->wt_api = connection->get_extension_api(connection); - - /* between 0-4: set the amount of verbosity to stderr */ - bzip_compressor->bz_verbosity = 0; - - /* - * between 1-9: set the block size to 100k x this number (compression - * only) - */ - bzip_compressor->bz_blocksize100k = 1; - - /* - * between 0-250: workFactor: see bzip2 manual. 0 is a reasonable - * default (compression only) - */ - bzip_compressor->bz_workfactor = 0; - - /* - * if nonzero, decompress using less memory, but slower (decompression - * only) - */ - bzip_compressor->bz_small = 0; - - return (connection->add_compressor( /* Load the compressor */ - connection, name, (WT_COMPRESSOR *)bzip_compressor, NULL)); -} - -/* - * wiredtiger_extension_init -- - * WiredTiger bzip2 compression extension. - */ -int -wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) -{ - int ret; - - (void)config; /* Unused parameters */ - - if ((ret = bzip2_add_compressor(connection, 0, "bzip2")) != 0) - return (ret); - if ((ret = bzip2_add_compressor(connection, 1, "bzip2-raw-test")) != 0) - return (ret); - return (0); -} diff --git a/src/docs/build-posix.dox b/src/docs/build-posix.dox index f61b199bff2..4889bf931c9 100644 --- a/src/docs/build-posix.dox +++ b/src/docs/build-posix.dox @@ -116,10 +116,6 @@ The WiredTiger software supports some additional configuration options: Configure WiredTiger to sleep and wait for a debugger to attach on failure. <b>DO NOT</b> configure this option in production environments. -@par \c --enable-bzip2 -Configure WiredTiger for <a href="http://www.bzip.org/">bzip2</a> -compression; see @ref compression for more information. - @par \c --enable-diagnostic Configure WiredTiger to perform various run-time diagnostic tests. <b>DO NOT</b> configure this option in production environments. diff --git a/src/docs/compression.dox b/src/docs/compression.dox index 56715e20752..0be96835760 100644 --- a/src/docs/compression.dox +++ b/src/docs/compression.dox @@ -1,36 +1,7 @@ /*! @m_page{{c,java},compression,Compressors} This section explains how to configure WiredTiger's builtin support for -the bzip2, lz4, snappy and zlib compression engines. - -@section compression_bzip2 Using bzip2 compression - -To use the builtin support for -<a href="http://www.bzip.org/">Julian Seward's bzip2</a> -compression, first check that bzip2 is installed in include and library -directories searched by the compiler. Once bzip2 is installed, you can -enable bzip2 using the \c --enable-bzip2 option to configure. - -If bzip2 is installed in a location not normally searched by the -compiler toolchain, you'll need to modify the \c CPPFLAGS and \c LDFLAGS -to indicate these locations. For example, with the bzip2 includes and -libraries installed in \c /usr/local/include and \c /usr/local/lib, you -would run configure with the following additional arguments: - -@code ---enable-bzip2 CPPFLAGS="-I/usr/local/include" LDFLAGS="-L/usr/local/include" -@endcode - -When opening the WiredTiger database, load the bzip2 shared library as -an extension. For example, with the WiredTiger library installed in -\c /usr/local/lib, you would use the following extension: - -@snippet ex_all.c Configure bzip2 extension - -Finally, when creating the WiredTiger object, set \c block_compressor -to \c bzip2: - -@snippet ex_all.c Create a bzip2 compressed table +the lz4, snappy and zlib compression engines. @section compression_lz4 Using LZ4 compression diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index e0239919f0b..e4d85003a1e 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -27,6 +27,13 @@ reclaimed when the call returns. The performance of this API may differ from earlier releases. </dd> +<dt>Bzip2 compression support</dt> +<dd> +Support for the bzip2 compression/decompression engine has been removed +from the WiredTiger release; remaining compression engines include LZ4, +snappy and zlib. +</dd> + </dl><hr> @section version_270 Upgrading to Version 2.7.0 diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox index 64e25978dd8..1f0d1533ac4 100644 --- a/src/docs/wtperf.dox +++ b/src/docs/wtperf.dox @@ -160,7 +160,7 @@ connection configuration string post-populate compact for LSM merging activity @par compression (string, default=none) compression extension. Allowed configuration values are: 'none', -'bzip', 'lz4', 'snappy', 'zlib' +'lz4', 'snappy', 'zlib' @par create (boolean, default=true) do population phase; false to use existing database @par database_count (unsigned int, default=1) diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 676f95d9b05..2d12abab248 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1006,9 +1006,9 @@ struct __wt_session { * @config{block_compressor, configure a compressor for file blocks. * Permitted values are \c "none" or custom compression engine name * created with WT_CONNECTION::add_compressor. If WiredTiger has - * builtin support for \c "bzip2"\, \c "snappy"\, \c "lz4" or \c "zlib" - * compression\, these names are also available. See @ref compression - * for more information., a string; default \c none.} + * builtin support for \c "snappy"\, \c "lz4" or \c "zlib" compression\, + * these names are also available. See @ref compression for more + * information., a string; default \c none.} * @config{cache_resident, do not ever evict the object's pages from * cache. Not compatible with LSM tables; see @ref * tuning_cache_resident for more information., a boolean flag; default @@ -1839,9 +1839,9 @@ struct __wt_connection { * @config{ compressor, configure a compressor * for log records. Permitted values are \c "none" or custom * compression engine name created with WT_CONNECTION::add_compressor. - * If WiredTiger has builtin support for \c "bzip2"\, \c "snappy"\, \c - * "lz4" or \c "zlib" compression\, these names are also available. See - * @ref compression for more information., a string; default \c none.} + * If WiredTiger has builtin support for \c "snappy"\, \c "lz4" or \c + * "zlib" compression\, these names are also available. See @ref + * compression for more information., a string; default \c none.} * @config{ enabled, enable logging subsystem., a * boolean flag; default \c false.} * @config{ file_max, the maximum size of log @@ -2308,11 +2308,11 @@ struct __wt_connection { * @config{ compressor, configure a compressor for log * records. Permitted values are \c "none" or custom compression engine name * created with WT_CONNECTION::add_compressor. If WiredTiger has builtin - * support for \c "bzip2"\, \c "snappy"\, \c "lz4" or \c "zlib" compression\, - * these names are also available. See @ref compression for more information., - * a string; default \c none.} - * @config{ enabled, enable - * logging subsystem., a boolean flag; default \c false.} + * support for \c "snappy"\, \c "lz4" or \c "zlib" compression\, these names are + * also available. See @ref compression for more information., a string; + * default \c none.} + * @config{ enabled, enable logging + * subsystem., a boolean flag; default \c false.} * @config{ file_max, the maximum size of log files., an * integer between 100KB and 2GB; default \c 100MB.} * @config{ path, the path to a directory into which the diff --git a/test/format/config.c b/test/format/config.c index 4a73dee21ee..77f07423e06 100644 --- a/test/format/config.c +++ b/test/format/config.c @@ -248,41 +248,47 @@ config_compression(const char *conf_name) return; /* - * Compression: choose something if compression wasn't specified. We - * used to verify the shared libraries existed but that's no longer - * robust, since it's possible to build compression libraries into the - * WiredTiger library. + * Compression: If compression wasn't specified, select a compression + * type from the list of built-in engines. */ - cstr = "none"; - if (strcmp(conf_name, "logging_compression") != 0 || g.c_logging != 0) + if (!config_is_perm(conf_name)) { + /* + * Listed percentages are only correct if all of the possible + * engines are compiled in. + */ switch (mmrand(NULL, 1, 20)) { - case 1: case 2: case 3: case 4: /* 20% no compression */ - break; - case 5: /* 5% bzip */ - cstr = "bzip"; - break; - case 6: /* 5% bzip-raw */ - cstr = "bzip-raw"; - break; - case 7: case 8: case 9: case 10: /* 20% lz4 */ +#ifdef HAVE_BUILTIN_EXTENSION_LZ4 + case 1: case 2: case 3: case 4: /* 20% lz4 */ cstr = "lz4"; break; - case 11: /* 5% lz4-no-raw */ + case 5: /* 5% lz4-no-raw */ cstr = "lz4-noraw"; break; - case 12: case 13: case 14: case 15: /* 20% snappy */ +#endif +#ifdef HAVE_BUILTIN_EXTENSION_SNAPPY + case 6: case 7: case 8: case 9: /* 30% snappy */ + case 10: case 11: cstr = "snappy"; break; - case 16: case 17: case 18: case 19: /* 20% zlib */ +#endif +#ifdef HAVE_BUILTIN_EXTENSION_ZLIB + case 12: case 13: case 14: case 15: /* 20% zlib */ cstr = "zlib"; break; - case 20: /* 5% zlib-no-raw */ + case 16: /* 5% zlib-no-raw */ cstr = "zlib-noraw"; break; +#endif + case 17: case 18: case 19: case 20: /* 20% no compression */ + default: + cstr = "none"; + break; } - (void)snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); - config_single(confbuf, 0); + (void)snprintf( + confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + config_single(confbuf, 0); + } } /* @@ -641,10 +647,6 @@ config_map_compression(const char *s, u_int *vp) { if (strcmp(s, "none") == 0) *vp = COMPRESS_NONE; - else if (strcmp(s, "bzip") == 0) - *vp = COMPRESS_BZIP; - else if (strcmp(s, "bzip-raw") == 0) - *vp = COMPRESS_BZIP_RAW; else if (strcmp(s, "lz4") == 0) *vp = COMPRESS_LZ4; else if (strcmp(s, "lz4-noraw") == 0) diff --git a/test/format/config.h b/test/format/config.h index a5190469c7f..d8b11b005d4 100644 --- a/test/format/config.h +++ b/test/format/config.h @@ -58,8 +58,7 @@ typedef struct { } CONFIG; #define COMPRESSION_LIST \ - "(none | bzip | bzip-raw | lz4 | lz4-noraw | lzo | none | " \ - "snappy | zlib | zlib-noraw)" + "(none | lz4 | lz4-noraw | snappy | zlib | zlib-noraw)" static CONFIG c[] = { { "abort", diff --git a/test/format/format.h b/test/format/format.h index bf4d1c0a277..41c9de3dd30 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -64,8 +64,6 @@ #define EXTPATH "../../ext/" /* Extensions path */ -#define BZIP_PATH \ - EXTPATH "compressors/bzip2/.libs/libwiredtiger_bzip2.so" #define LZ4_PATH \ EXTPATH "compressors/lz4/.libs/libwiredtiger_lz4.so" #define SNAPPY_PATH \ @@ -241,14 +239,12 @@ typedef struct { u_int c_checksum_flag; /* Checksum flag value */ #define COMPRESS_NONE 1 -#define COMPRESS_BZIP 2 -#define COMPRESS_BZIP_RAW 3 -#define COMPRESS_LZ4 4 -#define COMPRESS_LZ4_NO_RAW 5 -#define COMPRESS_LZO 6 -#define COMPRESS_SNAPPY 7 -#define COMPRESS_ZLIB 8 -#define COMPRESS_ZLIB_NO_RAW 9 +#define COMPRESS_LZ4 2 +#define COMPRESS_LZ4_NO_RAW 3 +#define COMPRESS_LZO 4 +#define COMPRESS_SNAPPY 5 +#define COMPRESS_ZLIB 6 +#define COMPRESS_ZLIB_NO_RAW 7 u_int c_compression_flag; /* Compression flag value */ u_int c_logging_compression_flag; /* Log compression flag value */ diff --git a/test/format/wts.c b/test/format/wts.c index 46b13f3461d..9d4d3fe5cb8 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -38,10 +38,6 @@ compressor(uint32_t compress_flag) switch (compress_flag) { case COMPRESS_NONE: return ("none"); - case COMPRESS_BZIP: - return ("bzip2"); - case COMPRESS_BZIP_RAW: - return ("bzip2-raw-test"); case COMPRESS_LZ4: return ("lz4"); case COMPRESS_LZ4_NO_RAW: @@ -206,9 +202,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) /* Extensions. */ p += snprintf(p, REMAIN(p, end), ",extensions=[" - "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", + "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", g.c_reverse ? REVERSE_PATH : "", - access(BZIP_PATH, R_OK) == 0 ? BZIP_PATH : "", access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", access(LZO_PATH, R_OK) == 0 ? LZO_PATH : "", access(ROTN_PATH, R_OK) == 0 ? ROTN_PATH : "", diff --git a/test/suite/test_compress01.py b/test/suite/test_compress01.py index e97953a53cd..94c748fc3e5 100644 --- a/test/suite/test_compress01.py +++ b/test/suite/test_compress01.py @@ -42,7 +42,6 @@ class test_compress01(wttest.WiredTigerTestCase): ('table', dict(uri='table:test_compress01')), ] compress = [ - ('bzip2', dict(compress='bzip2')), ('nop', dict(compress='nop')), ('snappy', dict(compress='snappy')), ('none', dict(compress=None)), diff --git a/test/suite/test_cursor08.py b/test/suite/test_cursor08.py index f7bd37a2a3c..1a379518224 100644 --- a/test/suite/test_cursor08.py +++ b/test/suite/test_cursor08.py @@ -47,7 +47,6 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess): ('reopen', dict(reopen=True)) ]) compress = check_scenarios([ - ('bzip2', dict(compress='bzip2')), ('nop', dict(compress='nop')), ('snappy', dict(compress='snappy')), ('zlib', dict(compress='zlib')), diff --git a/test/suite/test_encrypt01.py b/test/suite/test_encrypt01.py index 0521f7c8e8a..0f2782204d2 100644 --- a/test/suite/test_encrypt01.py +++ b/test/suite/test_encrypt01.py @@ -55,10 +55,8 @@ class test_encrypt01(wttest.WiredTigerTestCase): ('none', dict(log_compress=None, block_compress=None)), ('nop', dict(log_compress='nop', block_compress='nop')), ('lz4', dict(log_compress='lz4', block_compress='lz4')), - ('bzip2', dict(log_compress='bzip2', block_compress='bzip2')), ('snappy', dict(log_compress='snappy', block_compress='snappy')), ('zlib', dict(log_compress='zlib', block_compress='zlib')), - ('bzip2-none', dict(log_compress='bzip2', block_compress=None)), ('none-snappy', dict(log_compress=None, block_compress='snappy')), ('snappy-lz4', dict(log_compress='snappy', block_compress='lz4')), ] diff --git a/test/suite/test_txn07.py b/test/suite/test_txn07.py index 94c26990178..f74120e3590 100644 --- a/test/suite/test_txn07.py +++ b/test/suite/test_txn07.py @@ -64,7 +64,6 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): ] txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))] compress = [ - ('bzip2', dict(compress='bzip2')), ('nop', dict(compress='nop')), ('snappy', dict(compress='snappy')), ('zlib', dict(compress='zlib')), |