Added a compressor adaptor for Google's snappy compressor. refs #58

7 files changed, 179 insertions, 1 deletions

diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs
index 49489e8bbb7..409b10ed5d1 100644
--- a/build_posix/Make.subdirs
+++ b/build_posix/Make.subdirs
@@ -9,6 +9,7 @@ bench/tcbench
 examples/c
 ext/collators/reverse
 ext/compressors/bzip2_compress BZIP2
+ext/compressors/snappy_compress SNAPPY
 ext/compressors/nop_compress
 lang/python PYTHON
 test/format HAVE_BDB
diff --git a/build_posix/aclocal/options.m4 b/build_posix/aclocal/options.m4
index 90718756972..fd5ae600877 100644
--- a/build_posix/aclocal/options.m4
+++ b/build_posix/aclocal/options.m4
@@ -60,6 +60,17 @@ esac
 AC_MSG_RESULT($wt_cv_enable_python)
 AM_CONDITIONAL(PYTHON, test x$wt_cv_enable_python = xyes)
 
+AC_MSG_CHECKING(if --enable-snappy option specified)
+AC_ARG_ENABLE(snappy,
+	[AC_HELP_STRING([--enable-snappy],
+	    [Build the snappy compressor extension.])], r=$enableval, r=no)
+case "$r" in
+no)	wt_cv_enable_snappy=no;;
+*)	wt_cv_enable_snappy=yes;;
+esac
+AC_MSG_RESULT($wt_cv_enable_snappy)
+AM_CONDITIONAL([SNAPPY], [test x$wt_cv_enable_snappy = xyes])
+
 AH_TEMPLATE(HAVE_VERBOSE, [Define to 1 to support the Env.verbose_set method.])
 AC_MSG_CHECKING(if --enable-verbose option specified)
 AC_ARG_ENABLE(verbose,
diff --git a/docs/src/compression.dox b/docs/src/compression.dox
index 9c9a214eb66..696666dd0b9 100644
--- a/docs/src/compression.dox
+++ b/docs/src/compression.dox
@@ -1,6 +1,6 @@
 /*! @page compression Compression Engines
 
-This section explains how to use compression engines with WiredTiger, including the builtin support for bzip2.
+This section explains how to use compression engines with WiredTiger, including the builtin support for bzip2 and snappy.
 
 @todo describe how to use compression
  */
diff --git a/docs/src/install.dox b/docs/src/install.dox
index d88e22fcf14..95b07127272 100644
--- a/docs/src/install.dox
+++ b/docs/src/install.dox
@@ -119,6 +119,10 @@ Configure WiredTiger to perform various run-time diagnostic tests.
 @par \c --enable-python
 Build the WiredTiger <a href="http://www.python.org">Python</a> API.
 
+@par \c --enable-snappy
+Configure WiredTiger for <a href="http://code.google.com/p/snappy/">snappy</a>
+compression.
+
 @par \c --enable-verbose
 Configure WiredTiger to support the \c verbose configuration string to
 ::wiredtiger_open.
diff --git a/ext/compressors/snappy_compress/Makefile.am b/ext/compressors/snappy_compress/Makefile.am
new file mode 100644
index 00000000000..4592b986f75
--- /dev/null
+++ b/ext/compressors/snappy_compress/Makefile.am
@@ -0,0 +1,5 @@
+INCLUDES = -I$(top_builddir) -I$(top_srcdir)/src/include
+
+lib_LTLIBRARIES = snappy_compress.la
+snappy_compress_la_LDFLAGS = -avoid-version -module
+snappy_compress_la_LIBADD = -lsnappy
diff --git a/ext/compressors/snappy_compress/snappy_compress.c b/ext/compressors/snappy_compress/snappy_compress.c
new file mode 100644
index 00000000000..eb11cd5618e
--- /dev/null
+++ b/ext/compressors/snappy_compress/snappy_compress.c
@@ -0,0 +1,153 @@
+#include <errno.h>
+#include <snappy-c.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <wiredtiger.h>
+#include <wiredtiger_ext.h>
+
+WT_EXTENSION_API *wt_api;
+
+static int
+wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
+    const WT_ITEM *source, WT_ITEM *dest, int *);
+	
+static int
+wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
+    const WT_ITEM *source, WT_ITEM *dest);
+
+static WT_COMPRESSOR wt_snappy_compressor = {
+    wt_snappy_compress, wt_snappy_decompress };
+
+#define	__UNUSED(v)	((void)(v))
+
+int
+wiredtiger_extension_init(
+    WT_SESSION *session, WT_EXTENSION_API *api, const char *config)
+{
+	WT_CONNECTION *conn;
+
+	__UNUSED(config);
+
+	wt_api = api;
+	conn = session->connection;
+
+	return (conn->add_compressor(
+            conn, "snappy_compress", &wt_snappy_compressor, NULL));
+}
+
+/*
+ * wt_snappy_error --
+ *	Output an error message, and return a standard error code.
+ */
+static int
+wt_snappy_error(WT_SESSION *session, const char *call, snappy_status snret)
+{
+	const char *msg;
+
+	switch (snret) {
+	case SNAPPY_BUFFER_TOO_SMALL:
+		msg = "SNAPPY_BUFFER_TOO_SMALL";
+		break;
+	case SNAPPY_INVALID_INPUT:
+		msg = "SNAPPY_INVALID_INPUT";
+		break;
+	default:
+		msg = "unknown error";
+		break;
+	}
+
+	wiredtiger_err_printf(
+	    session, "snappy error: %s: %s: %d", call, msg, snret);
+	return (WT_ERROR);
+}
+
+/* Implementation of WT_COMPRESSOR for WT_CONNECTION::add_compressor. */
+static int
+wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
+    const WT_ITEM *src, WT_ITEM *dst, int *compression_failed)
+{
+	snappy_status snret;
+	int ret;
+	size_t snaplen;
+	void *buf;
+	char *snapbuf;
+	unsigned char *destp;
+
+        __UNUSED(compressor);
+
+	/* Snappy requires that the dest buffer be somewhat larger
+	 * than the source, but WT hands us a a dest buffer the same
+	 * size.  This leads to an unfortunate extra copy, but it
+	 * can't be helped with the current WT interface and snappy
+	 * API.  We use a scratch buffer for temp storage, it should
+	 * be much faster than malloc.
+	 */
+	snaplen = snappy_max_compressed_length(src->size);
+	buf = wiredtiger_scr_alloc(session, snaplen);
+	snapbuf = (char *)buf;
+
+	/* snaplen is an input and an output arg. */
+	snret = snappy_compress((char *)src->data, src->size,
+	    snapbuf, &snaplen);
+
+	if (snret == SNAPPY_OK) {
+		/* On decompression, snappy requires the exact
+		 * compressed buffer size (the current value of
+		 * snaplen).  WT does not preserve that, it rounds
+		 * the length up.  So we save the snaplen at the
+		 * beginning of the destination buffer.
+		 */
+		if (snaplen + sizeof(size_t) < dst->size) {
+			destp = (unsigned char *)dst->data;
+			*(size_t *)destp = snaplen;
+			destp += sizeof(size_t);
+			memcpy(destp, snapbuf, snaplen);
+			dst->size = snaplen + sizeof(size_t);
+			*compression_failed  = 0;
+		}
+		else
+			/* The compressor failed to produce a smaller result. */
+			*compression_failed = 1;
+
+		ret = 0;
+	}
+	else
+		ret = wt_snappy_error(session, "snappy_compress", snret);
+
+	wiredtiger_scr_free(session, buf);
+
+        return (ret);
+}
+
+static int
+wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
+    const WT_ITEM *src, WT_ITEM *dst)
+{
+	snappy_status snret;
+	int ret;
+	size_t dstlen;
+	size_t snaplen;
+
+        __UNUSED(compressor);
+
+	/* retrieve the saved length */
+	snaplen = *((size_t *)src->data);
+	if (snaplen + sizeof(size_t) > src->size) {
+		fprintf(stderr, "wt_snap_decompress: stored size exceeds buf size\n");
+		return (WT_ERROR);
+	}
+	dstlen = dst->size;
+	snret = snappy_uncompress(((char *)src->data) + sizeof(size_t), snaplen,
+	    (char *)dst->data, &dstlen);
+
+	if (snret == SNAPPY_OK) {
+		dst->size = dstlen;
+		ret = 0;
+	}
+	else
+		ret = wt_snappy_error(session, "snappy_uncompress", snret);
+
+        return (ret);
+}
+/* End implementation of WT_COMPRESSOR. */
diff --git a/test/suite/test_compress01.py b/test/suite/test_compress01.py
index 0afa73cf514..6754a0b7e20 100644
--- a/test/suite/test_compress01.py
+++ b/test/suite/test_compress01.py
@@ -120,6 +120,10 @@ class test_compress01_2_bz(test_compress01_base):
     def __init__(self, testname):
         test_compress01_base.__init__(self, testname, 'bzip2_compress', 'bz')
 
+class test_compress01_3_sn(test_compress01_base):
+    def __init__(self, testname):
+        test_compress01_base.__init__(self, testname, 'snappy_compress', 'sn')
+
 
 if __name__ == '__main__':
     wttest.run(test_compress01_base)