diff options
| -rw-r--r-- | build_posix/Make.subdirs | 1 | ||||
| -rw-r--r-- | build_posix/aclocal/options.m4 | 11 | ||||
| -rw-r--r-- | docs/src/compression.dox | 2 | ||||
| -rw-r--r-- | docs/src/install.dox | 4 | ||||
| -rw-r--r-- | ext/compressors/snappy_compress/Makefile.am | 5 | ||||
| -rw-r--r-- | ext/compressors/snappy_compress/snappy_compress.c | 153 | ||||
| -rw-r--r-- | test/suite/test_compress01.py | 4 |
7 files changed, 179 insertions, 1 deletions
diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs index 49489e8bbb7..409b10ed5d1 100644 --- a/build_posix/Make.subdirs +++ b/build_posix/Make.subdirs @@ -9,6 +9,7 @@ bench/tcbench examples/c ext/collators/reverse ext/compressors/bzip2_compress BZIP2 +ext/compressors/snappy_compress SNAPPY ext/compressors/nop_compress lang/python PYTHON test/format HAVE_BDB diff --git a/build_posix/aclocal/options.m4 b/build_posix/aclocal/options.m4 index 90718756972..fd5ae600877 100644 --- a/build_posix/aclocal/options.m4 +++ b/build_posix/aclocal/options.m4 @@ -60,6 +60,17 @@ esac AC_MSG_RESULT($wt_cv_enable_python) AM_CONDITIONAL(PYTHON, test x$wt_cv_enable_python = xyes) +AC_MSG_CHECKING(if --enable-snappy option specified) +AC_ARG_ENABLE(snappy, + [AC_HELP_STRING([--enable-snappy], + [Build the snappy compressor extension.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_snappy=no;; +*) wt_cv_enable_snappy=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_snappy) +AM_CONDITIONAL([SNAPPY], [test x$wt_cv_enable_snappy = xyes]) + AH_TEMPLATE(HAVE_VERBOSE, [Define to 1 to support the Env.verbose_set method.]) AC_MSG_CHECKING(if --enable-verbose option specified) AC_ARG_ENABLE(verbose, diff --git a/docs/src/compression.dox b/docs/src/compression.dox index 9c9a214eb66..696666dd0b9 100644 --- a/docs/src/compression.dox +++ b/docs/src/compression.dox @@ -1,6 +1,6 @@ /*! @page compression Compression Engines -This section explains how to use compression engines with WiredTiger, including the builtin support for bzip2. +This section explains how to use compression engines with WiredTiger, including the builtin support for bzip2 and snappy. @todo describe how to use compression */ diff --git a/docs/src/install.dox b/docs/src/install.dox index d88e22fcf14..95b07127272 100644 --- a/docs/src/install.dox +++ b/docs/src/install.dox @@ -119,6 +119,10 @@ Configure WiredTiger to perform various run-time diagnostic tests. @par \c --enable-python Build the WiredTiger <a href="http://www.python.org">Python</a> API. +@par \c --enable-snappy +Configure WiredTiger for <a href="http://code.google.com/p/snappy/">snappy</a> +compression. + @par \c --enable-verbose Configure WiredTiger to support the \c verbose configuration string to ::wiredtiger_open. diff --git a/ext/compressors/snappy_compress/Makefile.am b/ext/compressors/snappy_compress/Makefile.am new file mode 100644 index 00000000000..4592b986f75 --- /dev/null +++ b/ext/compressors/snappy_compress/Makefile.am @@ -0,0 +1,5 @@ +INCLUDES = -I$(top_builddir) -I$(top_srcdir)/src/include + +lib_LTLIBRARIES = snappy_compress.la +snappy_compress_la_LDFLAGS = -avoid-version -module +snappy_compress_la_LIBADD = -lsnappy diff --git a/ext/compressors/snappy_compress/snappy_compress.c b/ext/compressors/snappy_compress/snappy_compress.c new file mode 100644 index 00000000000..eb11cd5618e --- /dev/null +++ b/ext/compressors/snappy_compress/snappy_compress.c @@ -0,0 +1,153 @@ +#include <errno.h> +#include <snappy-c.h> +#include <stdlib.h> +#include <string.h> + +#include <wiredtiger.h> +#include <wiredtiger_ext.h> + +WT_EXTENSION_API *wt_api; + +static int +wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, + const WT_ITEM *source, WT_ITEM *dest, int *); + +static int +wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, + const WT_ITEM *source, WT_ITEM *dest); + +static WT_COMPRESSOR wt_snappy_compressor = { + wt_snappy_compress, wt_snappy_decompress }; + +#define __UNUSED(v) ((void)(v)) + +int +wiredtiger_extension_init( + WT_SESSION *session, WT_EXTENSION_API *api, const char *config) +{ + WT_CONNECTION *conn; + + __UNUSED(config); + + wt_api = api; + conn = session->connection; + + return (conn->add_compressor( + conn, "snappy_compress", &wt_snappy_compressor, NULL)); +} + +/* + * wt_snappy_error -- + * Output an error message, and return a standard error code. + */ +static int +wt_snappy_error(WT_SESSION *session, const char *call, snappy_status snret) +{ + const char *msg; + + switch (snret) { + case SNAPPY_BUFFER_TOO_SMALL: + msg = "SNAPPY_BUFFER_TOO_SMALL"; + break; + case SNAPPY_INVALID_INPUT: + msg = "SNAPPY_INVALID_INPUT"; + break; + default: + msg = "unknown error"; + break; + } + + wiredtiger_err_printf( + session, "snappy error: %s: %s: %d", call, msg, snret); + return (WT_ERROR); +} + +/* Implementation of WT_COMPRESSOR for WT_CONNECTION::add_compressor. */ +static int +wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, + const WT_ITEM *src, WT_ITEM *dst, int *compression_failed) +{ + snappy_status snret; + int ret; + size_t snaplen; + void *buf; + char *snapbuf; + unsigned char *destp; + + __UNUSED(compressor); + + /* Snappy requires that the dest buffer be somewhat larger + * than the source, but WT hands us a a dest buffer the same + * size. This leads to an unfortunate extra copy, but it + * can't be helped with the current WT interface and snappy + * API. We use a scratch buffer for temp storage, it should + * be much faster than malloc. + */ + snaplen = snappy_max_compressed_length(src->size); + buf = wiredtiger_scr_alloc(session, snaplen); + snapbuf = (char *)buf; + + /* snaplen is an input and an output arg. */ + snret = snappy_compress((char *)src->data, src->size, + snapbuf, &snaplen); + + if (snret == SNAPPY_OK) { + /* On decompression, snappy requires the exact + * compressed buffer size (the current value of + * snaplen). WT does not preserve that, it rounds + * the length up. So we save the snaplen at the + * beginning of the destination buffer. + */ + if (snaplen + sizeof(size_t) < dst->size) { + destp = (unsigned char *)dst->data; + *(size_t *)destp = snaplen; + destp += sizeof(size_t); + memcpy(destp, snapbuf, snaplen); + dst->size = snaplen + sizeof(size_t); + *compression_failed = 0; + } + else + /* The compressor failed to produce a smaller result. */ + *compression_failed = 1; + + ret = 0; + } + else + ret = wt_snappy_error(session, "snappy_compress", snret); + + wiredtiger_scr_free(session, buf); + + return (ret); +} + +static int +wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, + const WT_ITEM *src, WT_ITEM *dst) +{ + snappy_status snret; + int ret; + size_t dstlen; + size_t snaplen; + + __UNUSED(compressor); + + /* retrieve the saved length */ + snaplen = *((size_t *)src->data); + if (snaplen + sizeof(size_t) > src->size) { + fprintf(stderr, "wt_snap_decompress: stored size exceeds buf size\n"); + return (WT_ERROR); + } + dstlen = dst->size; + snret = snappy_uncompress(((char *)src->data) + sizeof(size_t), snaplen, + (char *)dst->data, &dstlen); + + if (snret == SNAPPY_OK) { + dst->size = dstlen; + ret = 0; + } + else + ret = wt_snappy_error(session, "snappy_uncompress", snret); + + return (ret); +} +/* End implementation of WT_COMPRESSOR. */ diff --git a/test/suite/test_compress01.py b/test/suite/test_compress01.py index 0afa73cf514..6754a0b7e20 100644 --- a/test/suite/test_compress01.py +++ b/test/suite/test_compress01.py @@ -120,6 +120,10 @@ class test_compress01_2_bz(test_compress01_base): def __init__(self, testname): test_compress01_base.__init__(self, testname, 'bzip2_compress', 'bz') +class test_compress01_3_sn(test_compress01_base): + def __init__(self, testname): + test_compress01_base.__init__(self, testname, 'snappy_compress', 'sn') + if __name__ == '__main__': wttest.run(test_compress01_base) |