summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Cahill <michael.cahill@mongodb.com>2016-02-01 16:32:17 +1100
committerMichael Cahill <michael.cahill@mongodb.com>2016-02-01 16:32:17 +1100
commitfd91b92b4bd8b9c804a309b620d377918a07adaf (patch)
treecdff6bc9df8acba7fb0e51cf5eccf640d5e33e7c
parenta0642fe126075cd85ba921c996f1e5746a03670b (diff)
parent2bf91e749c149ea9f3b61f78e88ac9f5d4a369eb (diff)
downloadmongo-fd91b92b4bd8b9c804a309b620d377918a07adaf.tar.gz
Merge branch 'develop' into wt-2356
-rw-r--r--bench/wtperf/config.c3
-rw-r--r--bench/wtperf/wtperf.h3
-rw-r--r--bench/wtperf/wtperf_opt.i2
-rw-r--r--build_posix/Make.subdirs1
-rw-r--r--build_posix/aclocal/options.m425
-rw-r--r--build_posix/configure.ac.in2
-rw-r--r--build_win/wiredtiger_config.h3
-rw-r--r--dist/api_data.py9
-rw-r--r--dist/extlist1
-rw-r--r--dist/s_define.list1
-rw-r--r--dist/s_string.ok2
-rw-r--r--examples/c/ex_all.c15
-rw-r--r--examples/c/ex_encrypt.c5
-rw-r--r--examples/c/ex_log.c37
-rw-r--r--examples/java/com/wiredtiger/examples/ex_all.java13
-rw-r--r--ext/compressors/bzip2/Makefile.am6
-rw-r--r--ext/compressors/bzip2/bzip2_compress.c415
-rw-r--r--ext/compressors/lz4/lz4_compress.c33
-rw-r--r--ext/compressors/snappy/snappy_compress.c37
-rw-r--r--src/block/block_map.c14
-rw-r--r--src/block/block_mgr.c129
-rw-r--r--src/block/block_open.c46
-rw-r--r--src/block/block_read.c23
-rw-r--r--src/block/block_slvg.c1
-rw-r--r--src/block/block_write.c23
-rw-r--r--src/btree/bt_handle.c4
-rw-r--r--src/btree/bt_split.c34
-rw-r--r--src/btree/bt_walk.c4
-rw-r--r--src/btree/row_srch.c31
-rw-r--r--src/conn/conn_api.c2
-rw-r--r--src/conn/conn_dhandle.c4
-rw-r--r--src/conn/conn_log.c41
-rw-r--r--src/conn/conn_sweep.c6
-rw-r--r--src/cursor/cur_log.c15
-rw-r--r--src/docs/build-posix.dox4
-rw-r--r--src/docs/compression.dox31
-rw-r--r--src/docs/upgrading.dox7
-rw-r--r--src/docs/wtperf.dox2
-rw-r--r--src/include/block.h46
-rw-r--r--src/include/btmem.h17
-rw-r--r--src/include/connection.h1
-rw-r--r--src/include/extern.h1
-rw-r--r--src/include/log.h88
-rw-r--r--src/include/log.i19
-rw-r--r--src/include/meta.h5
-rw-r--r--src/include/swap.h92
-rw-r--r--src/include/wiredtiger.in39
-rw-r--r--src/include/wt_internal.h5
-rw-r--r--src/log/log.c161
-rw-r--r--src/log/log_slot.c9
-rw-r--r--src/meta/meta_ckpt.c2
-rw-r--r--src/meta/meta_table.c12
-rw-r--r--src/session/session_dhandle.c5
-rw-r--r--src/support/crypto.c11
-rw-r--r--src/support/global.c36
-rw-r--r--src/support/hash_city.c51
-rw-r--r--src/txn/txn_ckpt.c4
-rw-r--r--src/txn/txn_log.c25
-rw-r--r--src/txn/txn_recover.c19
-rw-r--r--test/format/config.c87
-rw-r--r--test/format/config.h3
-rw-r--r--test/format/format.h16
-rw-r--r--test/format/t.c4
-rw-r--r--test/format/wts.c13
-rw-r--r--test/recovery/truncated-log.c16
-rw-r--r--test/suite/test_compress01.py1
-rw-r--r--test/suite/test_cursor08.py1
-rw-r--r--test/suite/test_encrypt01.py2
-rw-r--r--test/suite/test_txn07.py1
69 files changed, 850 insertions, 976 deletions
diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c
index d9a22f4708d..3cb20ff2b26 100644
--- a/bench/wtperf/config.c
+++ b/bench/wtperf/config.c
@@ -156,9 +156,6 @@ config_compress(CONFIG *cfg)
if (strcmp(s, "none") == 0) {
cfg->compress_ext = NULL;
cfg->compress_table = NULL;
- } else if (strcmp(s, "bzip") == 0) {
- cfg->compress_ext = BZIP_EXT;
- cfg->compress_table = BZIP_BLK;
} else if (strcmp(s, "lz4") == 0) {
cfg->compress_ext = LZ4_EXT;
cfg->compress_table = LZ4_BLK;
diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h
index 7dbe1822a26..f8d5a5c0bc9 100644
--- a/bench/wtperf/wtperf.h
+++ b/bench/wtperf/wtperf.h
@@ -73,9 +73,6 @@ typedef struct __truncate_queue_entry TRUNCATE_QUEUE_ENTRY;
#define EXTPATH "../../ext/compressors/" /* Extensions path */
#define BLKCMP_PFX ",block_compressor="
-#define BZIP_BLK BLKCMP_PFX "bzip2"
-#define BZIP_EXT \
- EXT_PFX EXTPATH "bzip2/.libs/libwiredtiger_bzip2.so" EXT_SFX
#define LZ4_BLK BLKCMP_PFX "lz4"
#define LZ4_EXT \
EXT_PFX EXTPATH "lz4/.libs/libwiredtiger_lz4.so" EXT_SFX
diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i
index 6dc2a6d5569..60bbaff56e5 100644
--- a/bench/wtperf/wtperf_opt.i
+++ b/bench/wtperf/wtperf_opt.i
@@ -97,7 +97,7 @@ DEF_OPT_AS_CONFIG_STRING(conn_config, "create",
DEF_OPT_AS_BOOL(compact, 0, "post-populate compact for LSM merging activity")
DEF_OPT_AS_STRING(compression, "none",
"compression extension. Allowed configuration values are: "
- "'none', 'bzip', 'lz4', 'snappy', 'zlib'")
+ "'none', 'lz4', 'snappy', 'zlib'")
DEF_OPT_AS_BOOL(create, 1,
"do population phase; false to use existing database")
DEF_OPT_AS_UINT32(database_count, 1,
diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs
index e2f128a48df..e1f8a05c613 100644
--- a/build_posix/Make.subdirs
+++ b/build_posix/Make.subdirs
@@ -6,7 +6,6 @@
# If the directory exists, it is added to AUTO_SUBDIRS.
# If a condition is included, the subdir is made conditional via AM_CONDITIONAL
ext/collators/reverse
-ext/compressors/bzip2 BZIP2
ext/compressors/lz4 LZ4
ext/compressors/nop
ext/compressors/snappy SNAPPY
diff --git a/build_posix/aclocal/options.m4 b/build_posix/aclocal/options.m4
index 01d08ce3d16..0fb49dbf1df 100644
--- a/build_posix/aclocal/options.m4
+++ b/build_posix/aclocal/options.m4
@@ -47,23 +47,6 @@ AM_CONDITIONAL([HAVE_BUILTIN_EXTENSION_ZLIB],
[test "$wt_cv_with_builtin_extension_zlib" = "yes"])
AC_MSG_RESULT($with_builtins)
-AC_MSG_CHECKING(if --enable-bzip2 option specified)
-AC_ARG_ENABLE(bzip2,
- [AS_HELP_STRING([--enable-bzip2],
- [Build the bzip2 compressor extension.])], r=$enableval, r=no)
-case "$r" in
-no) wt_cv_enable_bzip2=no;;
-*) wt_cv_enable_bzip2=yes;;
-esac
-AC_MSG_RESULT($wt_cv_enable_bzip2)
-if test "$wt_cv_enable_bzip2" = "yes"; then
- AC_CHECK_HEADER(bzlib.h,,
- [AC_MSG_ERROR([--enable-bzip2 requires bzlib.h])])
- AC_CHECK_LIB(bz2, BZ2_bzCompress,,
- [AC_MSG_ERROR([--enable-bzip2 requires bz2 library])])
-fi
-AM_CONDITIONAL([BZIP2], [test "$wt_cv_enable_bzip2" = "yes"])
-
AH_TEMPLATE(HAVE_DIAGNOSTIC, [Define to 1 for diagnostic tests.])
AC_MSG_CHECKING(if --enable-diagnostic option specified)
AC_ARG_ENABLE(diagnostic,
@@ -165,10 +148,8 @@ no) if test "$wt_cv_with_builtin_extension_snappy" = "yes"; then
esac
AC_MSG_RESULT($wt_cv_enable_snappy)
if test "$wt_cv_enable_snappy" = "yes"; then
- AC_LANG_PUSH([C++])
- AC_CHECK_HEADER(snappy.h,,
+ AC_CHECK_HEADER(snappy-c.h,,
[AC_MSG_ERROR([--enable-snappy requires snappy.h])])
- AC_LANG_POP([C++])
AC_CHECK_LIB(snappy, snappy_compress,,
[AC_MSG_ERROR([--enable-snappy requires snappy library])])
fi
@@ -193,10 +174,8 @@ no) if test "$wt_cv_with_builtin_extension_lz4" = "yes"; then
esac
AC_MSG_RESULT($wt_cv_enable_lz4)
if test "$wt_cv_enable_lz4" = "yes"; then
- AC_LANG_PUSH([C++])
AC_CHECK_HEADER(lz4.h,,
[AC_MSG_ERROR([--enable-lz4 requires lz4.h])])
- AC_LANG_POP([C++])
AC_CHECK_LIB(lz4, LZ4_compress_destSize,,
[AC_MSG_ERROR([--enable-lz4 requires lz4 library with LZ4_compress_destSize support])])
fi
@@ -212,10 +191,8 @@ no) wt_cv_enable_tcmalloc=no;;
esac
AC_MSG_RESULT($wt_cv_enable_tcmalloc)
if test "$wt_cv_enable_tcmalloc" = "yes"; then
- AC_LANG_PUSH([C++])
AC_CHECK_HEADER(gperftools/tcmalloc.h,,
[AC_MSG_ERROR([--enable-tcmalloc requires gperftools/tcmalloc.h])])
- AC_LANG_POP([C++])
AC_CHECK_LIB(tcmalloc, tc_calloc,,
[AC_MSG_ERROR([--enable-tcmalloc requires tcmalloc library])])
fi
diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in
index 5949fb0509c..875c8b436a8 100644
--- a/build_posix/configure.ac.in
+++ b/build_posix/configure.ac.in
@@ -31,7 +31,7 @@ LT_INIT([pic-only])
AC_SUBST([LIBTOOL_DEPS])
AC_PROG_CC(cc gcc)
-# AC_PROG_CXX(c++ g++)
+AC_PROG_CXX(c++ g++)
if test "$GCC" = "yes"; then
# The Solaris gcc compiler gets the additional -pthreads flag.
diff --git a/build_win/wiredtiger_config.h b/build_win/wiredtiger_config.h
index d1ed307cb85..33dbc9b724d 100644
--- a/build_win/wiredtiger_config.h
+++ b/build_win/wiredtiger_config.h
@@ -52,9 +52,6 @@
/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1
-/* Define to 1 if you have the `bz2' library (-lbz2). */
-/* #undef HAVE_LIBBZ2 */
-
/* Define to 1 if you have the `dl' library (-ldl). */
/* #undef HAVE_LIBDL */
diff --git a/dist/api_data.py b/dist/api_data.py
index e4055112d5e..c386c0b345d 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -136,8 +136,8 @@ file_config = format_meta + [
configure a compressor for file blocks. Permitted values are \c "none"
or custom compression engine name created with
WT_CONNECTION::add_compressor. If WiredTiger has builtin support for
- \c "bzip2", \c "snappy", \c "lz4" or \c "zlib" compression, these names
- are also available. See @ref compression for more information'''),
+ \c "snappy", \c "lz4" or \c "zlib" compression, these names are also
+ available. See @ref compression for more information'''),
Config('cache_resident', 'false', r'''
do not ever evict the object's pages from cache. Not compatible with
LSM tables; see @ref tuning_cache_resident for more information''',
@@ -422,9 +422,8 @@ connection_runtime_config = [
configure a compressor for log records. Permitted values are
\c "none" or custom compression engine name created with
WT_CONNECTION::add_compressor. If WiredTiger has builtin support
- for \c "bzip2", \c "snappy", \c "lz4" or \c "zlib" compression,
- these names are also available. See @ref compression for more
- information'''),
+ for \c "snappy", \c "lz4" or \c "zlib" compression, these names
+ are also available. See @ref compression for more information'''),
Config('enabled', 'false', r'''
enable logging subsystem''',
type='boolean'),
diff --git a/dist/extlist b/dist/extlist
index 874d21289d2..a5515642d48 100644
--- a/dist/extlist
+++ b/dist/extlist
@@ -2,7 +2,6 @@
# List of extension source files for WiredTiger library.
ext/collators/reverse/reverse_collator.c
-ext/compressors/bzip2/bzip2_compress.c
ext/compressors/nop/nop_compress.c
ext/compressors/snappy/snappy_compress.c
ext/compressors/zlib/zlib_compress.c
diff --git a/dist/s_define.list b/dist/s_define.list
index 8b0d9a0bdcd..ff24e3db822 100644
--- a/dist/s_define.list
+++ b/dist/s_define.list
@@ -55,3 +55,4 @@ WT_WITH_LOCK
__F
__WIREDTIGER_EXT_H_
__WIREDTIGER_H_
+__wt_bswap16
diff --git a/dist/s_string.ok b/dist/s_string.ok
index b2d5a453441..19fa27cd719 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -396,6 +396,7 @@ bool
boolean
br
breakpoint
+bswap
bt
btcur
btmem
@@ -409,6 +410,7 @@ builtin
builtins
bytelock
bytestring
+byteswap
byvalue
bzCompressEnd
bzCompressInit
diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c
index f7ee857a7c7..418c99ad6a3 100644
--- a/examples/c/ex_all.c
+++ b/examples/c/ex_all.c
@@ -591,13 +591,6 @@ session_ops(WT_SESSION *session)
* the code snippets, use #ifdef's to avoid running it.
*/
#ifdef MIGHT_NOT_RUN
- /*! [Create a bzip2 compressed table] */
- ret = session->create(session,
- "table:mytable",
- "block_compressor=bzip2,key_format=S,value_format=S");
- /*! [Create a bzip2 compressed table] */
- ret = session->drop(session, "table:mytable", NULL);
-
/*! [Create a lz4 compressed table] */
ret = session->create(session,
"table:mytable",
@@ -1084,14 +1077,6 @@ main(void)
* be installed, causing the open to fail. The documentation requires
* the code snippets, use #ifdef's to avoid running it.
*/
- /*! [Configure bzip2 extension] */
- ret = wiredtiger_open(home, NULL,
- "create,"
- "extensions=[/usr/local/lib/libwiredtiger_bzip2.so]", &conn);
- /*! [Configure bzip2 extension] */
- if (ret == 0)
- (void)conn->close(conn, NULL);
-
/*! [Configure lz4 extension] */
ret = wiredtiger_open(home, NULL,
"create,"
diff --git a/examples/c/ex_encrypt.c b/examples/c/ex_encrypt.c
index 425ee6b7287..c53a61c92ea 100644
--- a/examples/c/ex_encrypt.c
+++ b/examples/c/ex_encrypt.c
@@ -389,16 +389,15 @@ simple_walk_log(WT_SESSION *session)
{
WT_CURSOR *cursor;
WT_ITEM logrec_key, logrec_value;
- WT_LSN lsn;
uint64_t txnid;
- uint32_t fileid, opcount, optype, rectype;
+ uint32_t fileid, log_file, log_offset, opcount, optype, rectype;
int found, ret;
ret = session->open_cursor(session, "log:", NULL, NULL, &cursor);
found = 0;
while ((ret = cursor->next(cursor)) == 0) {
- ret = cursor->get_key(cursor, &lsn.file, &lsn.offset, &opcount);
+ ret = cursor->get_key(cursor, &log_file, &log_offset, &opcount);
ret = cursor->get_value(cursor, &txnid,
&rectype, &optype, &fileid, &logrec_key, &logrec_value);
diff --git a/examples/c/ex_log.c b/examples/c/ex_log.c
index cc6a3c46b93..78bd7e683cf 100644
--- a/examples/c/ex_log.c
+++ b/examples/c/ex_log.c
@@ -108,15 +108,15 @@ compare_tables(WT_SESSION *session, WT_SESSION *sess_copy)
/*! [log cursor walk] */
static void
-print_record(WT_LSN *lsn, uint32_t opcount,
+print_record(uint32_t log_file, uint32_t log_offset, uint32_t opcount,
uint32_t rectype, uint32_t optype, uint64_t txnid, uint32_t fileid,
WT_ITEM *key, WT_ITEM *value)
{
printf(
- "LSN [%" PRIu32 "][%" PRIu64 "].%" PRIu32
+ "LSN [%" PRIu32 "][%" PRIu32 "].%" PRIu32
": record type %" PRIu32 " optype %" PRIu32
" txnid %" PRIu64 " fileid %" PRIu32,
- lsn->file, (uint64_t)lsn->offset, opcount,
+ log_file, log_offset, opcount,
rectype, optype, txnid, fileid);
printf(" key size %zu value size %zu\n", key->size, value->size);
if (rectype == WT_LOGREC_MESSAGE)
@@ -131,10 +131,9 @@ static int
simple_walk_log(WT_SESSION *session, int count_min)
{
WT_CURSOR *cursor;
- WT_LSN lsn;
WT_ITEM logrec_key, logrec_value;
uint64_t txnid;
- uint32_t fileid, opcount, optype, rectype;
+ uint32_t fileid, log_file, log_offset, opcount, optype, rectype;
int count, ret;
/*! [log cursor open] */
@@ -145,14 +144,14 @@ simple_walk_log(WT_SESSION *session, int count_min)
while ((ret = cursor->next(cursor)) == 0) {
count++;
/*! [log cursor get_key] */
- ret = cursor->get_key(cursor, &lsn.file, &lsn.offset, &opcount);
+ ret = cursor->get_key(cursor, &log_file, &log_offset, &opcount);
/*! [log cursor get_key] */
/*! [log cursor get_value] */
ret = cursor->get_value(cursor, &txnid,
&rectype, &optype, &fileid, &logrec_key, &logrec_value);
/*! [log cursor get_value] */
- print_record(&lsn, opcount,
+ print_record(log_file, log_offset, opcount,
rectype, optype, txnid, fileid, &logrec_key, &logrec_value);
}
if (ret == WT_NOTFOUND)
@@ -173,11 +172,11 @@ walk_log(WT_SESSION *session)
{
WT_CONNECTION *wt_conn2;
WT_CURSOR *cursor, *cursor2;
- WT_LSN lsn, lsnsave;
WT_ITEM logrec_key, logrec_value;
WT_SESSION *session2;
uint64_t txnid;
uint32_t fileid, opcount, optype, rectype;
+ uint32_t log_file, log_offset, save_file, save_offset;
int first, i, in_txn, ret;
ret = setup_copy(&wt_conn2, &session2);
@@ -186,21 +185,23 @@ walk_log(WT_SESSION *session)
i = 0;
in_txn = 0;
txnid = 0;
- memset(&lsnsave, 0, sizeof(lsnsave));
+ save_file = save_offset = 0;
while ((ret = cursor->next(cursor)) == 0) {
- ret = cursor->get_key(cursor, &lsn.file, &lsn.offset, &opcount);
+ ret = cursor->get_key(cursor, &log_file, &log_offset, &opcount);
/*
* Save one of the LSNs we get back to search for it
* later. Pick a later one because we want to walk from
* that LSN to the end (where the multi-step transaction
* was performed). Just choose the record that is MAX_KEYS.
*/
- if (++i == MAX_KEYS)
- lsnsave = lsn;
+ if (++i == MAX_KEYS) {
+ save_file = log_file;
+ save_offset = log_offset;
+ }
ret = cursor->get_value(cursor, &txnid, &rectype,
&optype, &fileid, &logrec_key, &logrec_value);
- print_record(&lsn, opcount,
+ print_record(log_file, log_offset, opcount,
rectype, optype, txnid, fileid, &logrec_key, &logrec_value);
/*
@@ -245,7 +246,7 @@ walk_log(WT_SESSION *session)
ret = cursor->reset(cursor);
/*! [log cursor set_key] */
- cursor->set_key(cursor, lsnsave.file, lsnsave.offset, 0);
+ cursor->set_key(cursor, save_file, save_offset, 0);
/*! [log cursor set_key] */
/*! [log cursor search] */
ret = cursor->search(cursor);
@@ -256,11 +257,11 @@ walk_log(WT_SESSION *session)
*/
first = 1;
while ((ret = cursor->get_key(cursor,
- &lsn.file, &lsn.offset, &opcount)) == 0) {
+ &log_file, &log_offset, &opcount)) == 0) {
if (first) {
first = 0;
- if (lsnsave.file != lsn.file ||
- lsnsave.offset != lsn.offset) {
+ if (save_file != log_file ||
+ save_offset != log_offset) {
fprintf(stderr,
"search returned the wrong LSN\n");
exit (1);
@@ -269,7 +270,7 @@ walk_log(WT_SESSION *session)
ret = cursor->get_value(cursor, &txnid, &rectype,
&optype, &fileid, &logrec_key, &logrec_value);
- print_record(&lsn, opcount,
+ print_record(log_file, log_offset, opcount,
rectype, optype, txnid, fileid, &logrec_key, &logrec_value);
ret = cursor->next(cursor);
diff --git a/examples/java/com/wiredtiger/examples/ex_all.java b/examples/java/com/wiredtiger/examples/ex_all.java
index 153f12d3e27..09db8e0fd56 100644
--- a/examples/java/com/wiredtiger/examples/ex_all.java
+++ b/examples/java/com/wiredtiger/examples/ex_all.java
@@ -516,12 +516,6 @@ session_ops(Session session)
* the code snippets, use if (false) to avoid running it.
*/
if (false) { // MIGHT_NOT_RUN
- /*! [Create a bzip2 compressed table] */
- ret = session.create("table:mytable",
- "block_compressor=bzip2,key_format=S,value_format=S");
- /*! [Create a bzip2 compressed table] */
- ret = session.drop("table:mytable", null);
-
/*! [Create a lz4 compressed table] */
ret = session.create("table:mytable",
"block_compressor=lz4,key_format=S,value_format=S");
@@ -899,13 +893,6 @@ allExample()
* be installed, causing the open to fail. The documentation requires
* the code snippets, use if (false) to avoid running it.
*/
- /*! [Configure bzip2 extension] */
- conn = wiredtiger.open(home,
- "create," +
- "extensions=[/usr/local/lib/libwiredtiger_bzip2.so]");
- /*! [Configure bzip2 extension] */
- conn.close(null);
-
/*! [Configure lz4 extension] */
conn = wiredtiger.open(home,
"create," +
diff --git a/ext/compressors/bzip2/Makefile.am b/ext/compressors/bzip2/Makefile.am
deleted file mode 100644
index 0aedc2efd80..00000000000
--- a/ext/compressors/bzip2/Makefile.am
+++ /dev/null
@@ -1,6 +0,0 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
-
-lib_LTLIBRARIES = libwiredtiger_bzip2.la
-libwiredtiger_bzip2_la_SOURCES = bzip2_compress.c
-libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module
-libwiredtiger_bzip2_la_LIBADD = -lbz2
diff --git a/ext/compressors/bzip2/bzip2_compress.c b/ext/compressors/bzip2/bzip2_compress.c
deleted file mode 100644
index 845107c609f..00000000000
--- a/ext/compressors/bzip2/bzip2_compress.c
+++ /dev/null
@@ -1,415 +0,0 @@
-/*-
- * Public Domain 2014-2016 MongoDB, Inc.
- * Public Domain 2008-2014 WiredTiger, Inc.
- *
- * This is free and unencumbered software released into the public domain.
- *
- * Anyone is free to copy, modify, publish, use, compile, sell, or
- * distribute this software, either in source code form or as a compiled
- * binary, for any purpose, commercial or non-commercial, and by any
- * means.
- *
- * In jurisdictions that recognize copyright laws, the author or authors
- * of this software dedicate any and all copyright interest in the
- * software to the public domain. We make this dedication for the benefit
- * of the public at large and to the detriment of our heirs and
- * successors. We intend this dedication to be an overt act of
- * relinquishment in perpetuity of all present and future rights to this
- * software under copyright law.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <bzlib.h>
-#include <errno.h>
-#include <inttypes.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <wiredtiger.h>
-#include <wiredtiger_ext.h>
-
-/* Local compressor structure. */
-typedef struct {
- WT_COMPRESSOR compressor; /* Must come first */
-
- WT_EXTENSION_API *wt_api; /* Extension API */
-
- int bz_verbosity; /* Configuration */
- int bz_blocksize100k;
- int bz_workfactor;
- int bz_small;
-} BZIP_COMPRESSOR;
-
-/*
- * Bzip gives us a cookie to pass to the underlying allocation functions; we
- * we need two handles, package them up.
- */
-typedef struct {
- WT_COMPRESSOR *compressor;
- WT_SESSION *session;
-} BZIP_OPAQUE;
-
-/*
- * bzip2_error --
- * Output an error message, and return a standard error code.
- */
-static int
-bzip2_error(
- WT_COMPRESSOR *compressor, WT_SESSION *session, const char *call, int bzret)
-{
- WT_EXTENSION_API *wt_api;
- const char *msg;
-
- wt_api = ((BZIP_COMPRESSOR *)compressor)->wt_api;
-
- switch (bzret) {
- case BZ_MEM_ERROR:
- msg = "BZ_MEM_ERROR";
- break;
- case BZ_OUTBUFF_FULL:
- msg = "BZ_OUTBUFF_FULL";
- break;
- case BZ_SEQUENCE_ERROR:
- msg = "BZ_SEQUENCE_ERROR";
- break;
- case BZ_PARAM_ERROR:
- msg = "BZ_PARAM_ERROR";
- break;
- case BZ_DATA_ERROR:
- msg = "BZ_DATA_ERROR";
- break;
- case BZ_DATA_ERROR_MAGIC:
- msg = "BZ_DATA_ERROR_MAGIC";
- break;
- case BZ_IO_ERROR:
- msg = "BZ_IO_ERROR";
- break;
- case BZ_UNEXPECTED_EOF:
- msg = "BZ_UNEXPECTED_EOF";
- break;
- case BZ_CONFIG_ERROR:
- msg = "BZ_CONFIG_ERROR";
- break;
- default:
- msg = "unknown error";
- break;
- }
-
- (void)wt_api->err_printf(wt_api, session,
- "bzip2 error: %s: %s: %d", call, msg, bzret);
- return (WT_ERROR);
-}
-
-/*
- * bzalloc --
- * Allocate scratch buffers.
- */
-static void *
-bzalloc(void *cookie, int number, int size)
-{
- BZIP_OPAQUE *opaque;
- WT_EXTENSION_API *wt_api;
-
- opaque = cookie;
- wt_api = ((BZIP_COMPRESSOR *)opaque->compressor)->wt_api;
- return (wt_api->scr_alloc(
- wt_api, opaque->session, (size_t)(number * size)));
-}
-
-/*
- * bzfree --
- * Free scratch buffers.
- */
-static void
-bzfree(void *cookie, void *p)
-{
- BZIP_OPAQUE *opaque;
- WT_EXTENSION_API *wt_api;
-
- opaque = cookie;
- wt_api = ((BZIP_COMPRESSOR *)opaque->compressor)->wt_api;
- wt_api->scr_free(wt_api, opaque->session, p);
-}
-
-/*
- * bzip2_compress --
- * WiredTiger bzip2 compression.
- */
-static int
-bzip2_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
- uint8_t *src, size_t src_len,
- uint8_t *dst, size_t dst_len,
- size_t *result_lenp, int *compression_failed)
-{
- BZIP_COMPRESSOR *bzip_compressor;
- BZIP_OPAQUE opaque;
- bz_stream bz;
- int ret;
-
- bzip_compressor = (BZIP_COMPRESSOR *)compressor;
-
- memset(&bz, 0, sizeof(bz));
- bz.bzalloc = bzalloc;
- bz.bzfree = bzfree;
- opaque.compressor = compressor;
- opaque.session = session;
- bz.opaque = &opaque;
-
- if ((ret = BZ2_bzCompressInit(&bz,
- bzip_compressor->bz_blocksize100k,
- bzip_compressor->bz_verbosity,
- bzip_compressor->bz_workfactor)) != BZ_OK)
- return (bzip2_error(
- compressor, session, "BZ2_bzCompressInit", ret));
-
- bz.next_in = (char *)src;
- bz.avail_in = (uint32_t)src_len;
- bz.next_out = (char *)dst;
- bz.avail_out = (uint32_t)dst_len;
- if ((ret = BZ2_bzCompress(&bz, BZ_FINISH)) == BZ_STREAM_END) {
- *compression_failed = 0;
- *result_lenp = dst_len - bz.avail_out;
- } else
- *compression_failed = 1;
-
- if ((ret = BZ2_bzCompressEnd(&bz)) != BZ_OK)
- return (
- bzip2_error(compressor, session, "BZ2_bzCompressEnd", ret));
-
- return (0);
-}
-
-/*
- * __bzip2_compress_raw_random --
- * Return a 32-bit pseudo-random number.
- *
- * This is an implementation of George Marsaglia's multiply-with-carry pseudo-
- * random number generator. Computationally fast, with reasonable randomness
- * properties.
- */
-static uint32_t
-__bzip2_compress_raw_random(void)
-{
- static uint32_t m_w = 521288629;
- static uint32_t m_z = 362436069;
-
- m_z = 36969 * (m_z & 65535) + (m_z >> 16);
- m_w = 18000 * (m_w & 65535) + (m_w >> 16);
- return (m_z << 16) + (m_w & 65535);
-}
-
-/*
- * bzip2_compress_raw --
- * Test function for the test/format utility.
- */
-static int
-bzip2_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session,
- size_t page_max, int split_pct, size_t extra,
- uint8_t *src, uint32_t *offsets, uint32_t slots,
- uint8_t *dst, size_t dst_len, int final,
- size_t *result_lenp, uint32_t *result_slotsp)
-{
- uint32_t take, twenty_pct;
- int compression_failed, ret;
-
- (void)page_max; /* Unused parameters */
- (void)split_pct;
- (void)extra;
- (void)final;
-
- /*
- * This function is used by the test/format utility to test the
- * WT_COMPRESSOR::compress_raw functionality.
- *
- * I'm trying to mimic how a real application is likely to behave: if
- * it's a small number of slots, we're not going to take them because
- * they aren't worth compressing. In all likelihood, that's going to
- * be because the btree is wrapping up a page, but that's OK, that is
- * going to happen a lot. In addition, add a 2% chance of not taking
- * anything at all just because we don't want to take it. Otherwise,
- * select between 80 and 100% of the slots and compress them, stepping
- * down by 5 slots at a time until something works.
- */
- take = slots;
- if (take < 10 || __bzip2_compress_raw_random() % 100 < 2)
- take = 0;
- else {
- twenty_pct = (slots / 10) * 2;
- if (twenty_pct < slots)
- take -= __bzip2_compress_raw_random() % twenty_pct;
-
- for (;;) {
- if ((ret = bzip2_compress(compressor, session,
- src, offsets[take],
- dst, dst_len,
- result_lenp, &compression_failed)) != 0)
- return (ret);
- if (!compression_failed)
- break;
- if (take < 10) {
- take = 0;
- break;
- }
- take -= 5;
- }
- }
-
- *result_slotsp = take;
- if (take == 0)
- *result_lenp = 0;
-
-#if 0
- fprintf(stderr,
- "bzip2_compress_raw (%s): page_max %" PRIuMAX
- ", split_pct %u, extra %" PRIuMAX
- ", slots %" PRIu32 ", take %" PRIu32 ": %" PRIu32 " -> %"
- PRIuMAX "\n",
- final ? "final" : "not final",
- (uintmax_t)page_max, split_pct, (uintmax_t)extra,
- slots, take, offsets[take], (uintmax_t)*result_lenp);
-#endif
- return (take == 0 ? EAGAIN : 0);
-}
-
-/*
- * bzip2_decompress --
- * WiredTiger bzip2 decompression.
- */
-static int
-bzip2_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
- uint8_t *src, size_t src_len,
- uint8_t *dst, size_t dst_len,
- size_t *result_lenp)
-{
- BZIP_COMPRESSOR *bzip_compressor;
- BZIP_OPAQUE opaque;
- bz_stream bz;
- int ret, tret;
-
- bzip_compressor = (BZIP_COMPRESSOR *)compressor;
-
- memset(&bz, 0, sizeof(bz));
- bz.bzalloc = bzalloc;
- bz.bzfree = bzfree;
- opaque.compressor = compressor;
- opaque.session = session;
- bz.opaque = &opaque;
-
- if ((ret = BZ2_bzDecompressInit(&bz,
- bzip_compressor->bz_small, bzip_compressor->bz_verbosity)) != BZ_OK)
- return (bzip2_error(
- compressor, session, "BZ2_bzDecompressInit", ret));
-
- bz.next_in = (char *)src;
- bz.avail_in = (uint32_t)src_len;
- bz.next_out = (char *)dst;
- bz.avail_out = (uint32_t)dst_len;
- if ((ret = BZ2_bzDecompress(&bz)) == BZ_STREAM_END) {
- *result_lenp = dst_len - bz.avail_out;
- ret = 0;
- } else {
- /*
- * If BZ2_bzDecompress returns 0, it expects there to be more
- * data available. There isn't, so treat this as an error.
- */
- if (ret == 0)
- ret = BZ_DATA_ERROR;
- (void)bzip2_error(compressor, session, "BZ2_bzDecompress", ret);
- }
-
- if ((tret = BZ2_bzDecompressEnd(&bz)) != BZ_OK)
- return (bzip2_error(
- compressor, session, "BZ2_bzDecompressEnd", tret));
-
- return (ret == 0 ?
- 0 : bzip2_error(compressor, session, "BZ2_bzDecompressEnd", ret));
-}
-
-/*
- * bzip2_terminate --
- * WiredTiger bzip2 compression termination.
- */
-static int
-bzip2_terminate(WT_COMPRESSOR *compressor, WT_SESSION *session)
-{
- (void)session; /* Unused parameters */
-
- free(compressor);
- return (0);
-}
-
-/*
- * bzip2_add_compressor --
- * Add a bzip2 compressor.
- */
-static int
-bzip2_add_compressor(WT_CONNECTION *connection, int raw, const char *name)
-{
- BZIP_COMPRESSOR *bzip_compressor;
-
- /*
- * There are two almost identical bzip2 compressors: one supporting raw
- * compression (used by test/format to test raw compression), the other
- * without raw compression, that might be useful for real applications.
- */
- if ((bzip_compressor = calloc(1, sizeof(BZIP_COMPRESSOR))) == NULL)
- return (errno);
-
- bzip_compressor->compressor.compress = bzip2_compress;
- bzip_compressor->
- compressor.compress_raw = raw ? bzip2_compress_raw : NULL;
- bzip_compressor->compressor.decompress = bzip2_decompress;
- bzip_compressor->compressor.pre_size = NULL;
- bzip_compressor->compressor.terminate = bzip2_terminate;
-
- bzip_compressor->wt_api = connection->get_extension_api(connection);
-
- /* between 0-4: set the amount of verbosity to stderr */
- bzip_compressor->bz_verbosity = 0;
-
- /*
- * between 1-9: set the block size to 100k x this number (compression
- * only)
- */
- bzip_compressor->bz_blocksize100k = 1;
-
- /*
- * between 0-250: workFactor: see bzip2 manual. 0 is a reasonable
- * default (compression only)
- */
- bzip_compressor->bz_workfactor = 0;
-
- /*
- * if nonzero, decompress using less memory, but slower (decompression
- * only)
- */
- bzip_compressor->bz_small = 0;
-
- return (connection->add_compressor( /* Load the compressor */
- connection, name, (WT_COMPRESSOR *)bzip_compressor, NULL));
-}
-
-/*
- * wiredtiger_extension_init --
- * WiredTiger bzip2 compression extension.
- */
-int
-wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config)
-{
- int ret;
-
- (void)config; /* Unused parameters */
-
- if ((ret = bzip2_add_compressor(connection, 0, "bzip2")) != 0)
- return (ret);
- if ((ret = bzip2_add_compressor(connection, 1, "bzip2-raw-test")) != 0)
- return (ret);
- return (0);
-}
diff --git a/ext/compressors/lz4/lz4_compress.c b/ext/compressors/lz4/lz4_compress.c
index d070dc3fb79..3e6609beb80 100644
--- a/ext/compressors/lz4/lz4_compress.c
+++ b/ext/compressors/lz4/lz4_compress.c
@@ -31,17 +31,7 @@
#include <stdlib.h>
#include <string.h>
-#include <wiredtiger.h>
-#include <wiredtiger_ext.h>
-
-/*
- * We need to include the configuration file to detect whether this extension
- * is being built into the WiredTiger library.
- */
-#include "wiredtiger_config.h"
-#ifdef _MSC_VER
-#define inline __inline
-#endif
+#include <wt_internal.h>
/* Local compressor structure. */
typedef struct {
@@ -73,6 +63,24 @@ typedef struct {
} LZ4_PREFIX;
/*
+ * lz4_prefix_swap --
+ * The additional information is written in little-endian format, handle
+ * the conversion.
+ */
+static inline void
+lz4_prefix_swap(LZ4_PREFIX *prefix)
+{
+#ifdef WORDS_BIGENDIAN
+ prefix->compressed_len = __wt_bswap32(prefix->compressed_len);
+ prefix->uncompressed_len = __wt_bswap32(prefix->uncompressed_len);
+ prefix->useful_len = __wt_bswap32(prefix->useful_len);
+ prefix->unused = __wt_bswap32(prefix->unused);
+#else
+ WT_UNUSED(prefix);
+#endif
+}
+
+/*
* lz4_error --
* Output an error message, and return a standard error code.
*/
@@ -119,6 +127,7 @@ lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
prefix.uncompressed_len = (uint32_t)src_len;
prefix.useful_len = (uint32_t)src_len;
prefix.unused = 0;
+ lz4_prefix_swap(&prefix);
memcpy(dst, &prefix, sizeof(LZ4_PREFIX));
*result_lenp = (size_t)lz4_len + sizeof(LZ4_PREFIX);
@@ -154,6 +163,7 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
* decompressed bytes to return from the start of the source buffer.
*/
memcpy(&prefix, src, sizeof(LZ4_PREFIX));
+ lz4_prefix_swap(&prefix);
/*
* Decompress, starting after the prefix bytes. Use safe decompression:
@@ -268,6 +278,7 @@ lz4_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session,
prefix.uncompressed_len = (uint32_t)sourceSize;
prefix.useful_len = offsets[slot];
prefix.unused = 0;
+ lz4_prefix_swap(&prefix);
memcpy(dst, &prefix, sizeof(LZ4_PREFIX));
*result_slotsp = slot;
diff --git a/ext/compressors/snappy/snappy_compress.c b/ext/compressors/snappy/snappy_compress.c
index b5a347fce81..d1bd2e1813c 100644
--- a/ext/compressors/snappy/snappy_compress.c
+++ b/ext/compressors/snappy/snappy_compress.c
@@ -31,14 +31,7 @@
#include <stdlib.h>
#include <string.h>
-#include <wiredtiger.h>
-#include <wiredtiger_ext.h>
-
-/*
- * We need to include the configuration file to detect whether this extension
- * is being built into the WiredTiger library.
- */
-#include "wiredtiger_config.h"
+#include <wt_internal.h>
/* Local compressor structure. */
typedef struct {
@@ -103,16 +96,22 @@ wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
snret = snappy_compress((char *)src, src_len, snapbuf, &snaplen);
if (snret == SNAPPY_OK) {
- /*
- * On decompression, snappy requires the exact compressed byte
- * count (the current value of snaplen). WiredTiger does not
- * preserve that value, so save snaplen at the beginning of the
- * destination buffer.
- */
if (snaplen + sizeof(size_t) < src_len) {
- *(size_t *)dst = snaplen;
*result_lenp = snaplen + sizeof(size_t);
*compression_failed = 0;
+
+ /*
+ * On decompression, snappy requires an exact compressed
+ * byte count (the current value of snaplen). WiredTiger
+ * does not preserve that value, so save snaplen at the
+ * beginning of the destination buffer.
+ *
+ * Store the value in little-endian format.
+ */
+#ifdef WORDS_BIGENDIAN
+ snaplen = __wt_bswap64(snaplen);
+#endif
+ *(size_t *)dst = snaplen;
} else
/* The compressor failed to produce a smaller result. */
*compression_failed = 1;
@@ -137,8 +136,14 @@ wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session,
wt_api = ((SNAPPY_COMPRESSOR *)compressor)->wt_api;
- /* retrieve the saved length */
+ /*
+ * Retrieve the saved length, handling little- to big-endian conversion
+ * as necessary.
+ */
snaplen = *(size_t *)src;
+#ifdef WORDS_BIGENDIAN
+ snaplen = __wt_bswap64(snaplen);
+#endif
if (snaplen + sizeof(size_t) > src_len) {
(void)wt_api->err_printf(wt_api,
session,
diff --git a/src/block/block_map.c b/src/block/block_map.c
index 3d04a492269..b60623a37d8 100644
--- a/src/block/block_map.c
+++ b/src/block/block_map.c
@@ -20,6 +20,19 @@ __wt_block_map(
*(void **)mapp = NULL;
*maplenp = 0;
+#ifdef WORDS_BIGENDIAN
+ /*
+ * The underlying objects are little-endian, mapping objects isn't
+ * currently supported on big-endian systems.
+ */
+ WT_UNUSED(session);
+ WT_UNUSED(block);
+ WT_UNUSED(mappingcookie);
+#else
+ /* Map support is configurable. */
+ if (!S2C(session)->mmap)
+ return (0);
+
/*
* Turn off mapping when verifying the file, because we can't perform
* checksum validation of mapped segments, and verify has to checksum
@@ -48,6 +61,7 @@ __wt_block_map(
* Ignore errors, we'll read the file through the cache if map fails.
*/
(void)__wt_mmap(session, block->fh, mapp, maplenp, mappingcookie);
+#endif
return (0);
}
diff --git a/src/block/block_mgr.c b/src/block/block_mgr.c
index 6e2dc775362..dceaae8bb99 100644
--- a/src/block/block_mgr.c
+++ b/src/block/block_mgr.c
@@ -69,18 +69,6 @@ __bm_checkpoint(WT_BM *bm,
}
/*
- * __bm_sync --
- * Flush a file to disk.
- */
-static int
-__bm_sync(WT_BM *bm, WT_SESSION_IMPL *session, bool async)
-{
- return (async ?
- __wt_fsync_async(session, bm->block->fh) :
- __wt_fsync(session, bm->block->fh));
-}
-
-/*
* __bm_checkpoint_load --
* Load a checkpoint.
*/
@@ -89,10 +77,6 @@ __bm_checkpoint_load(WT_BM *bm, WT_SESSION_IMPL *session,
const uint8_t *addr, size_t addr_size,
uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint)
{
- WT_CONNECTION_IMPL *conn;
-
- conn = S2C(session);
-
/* If not opening a checkpoint, we're opening the live system. */
bm->is_live = !checkpoint;
WT_RET(__wt_block_checkpoint_load(session, bm->block,
@@ -103,9 +87,8 @@ __bm_checkpoint_load(WT_BM *bm, WT_SESSION_IMPL *session,
* Read-only objects are optionally mapped into memory instead
* of being read into cache buffers.
*/
- if (conn->mmap)
- WT_RET(__wt_block_map(session, bm->block,
- &bm->map, &bm->maplen, &bm->mappingcookie));
+ WT_RET(__wt_block_map(session,
+ bm->block, &bm->map, &bm->maplen, &bm->mappingcookie));
/*
* If this handle is for a checkpoint, that is, read-only, there
@@ -168,13 +151,13 @@ __bm_close(WT_BM *bm, WT_SESSION_IMPL *session)
}
/*
- * __bm_compact_start --
- * Start a block manager compaction.
+ * __bm_compact_end --
+ * End a block manager compaction.
*/
static int
-__bm_compact_start(WT_BM *bm, WT_SESSION_IMPL *session)
+__bm_compact_end(WT_BM *bm, WT_SESSION_IMPL *session)
{
- return (__wt_block_compact_start(session, bm->block));
+ return (__wt_block_compact_end(session, bm->block));
}
/*
@@ -200,13 +183,13 @@ __bm_compact_skip(WT_BM *bm, WT_SESSION_IMPL *session, bool *skipp)
}
/*
- * __bm_compact_end --
- * End a block manager compaction.
+ * __bm_compact_start --
+ * Start a block manager compaction.
*/
static int
-__bm_compact_end(WT_BM *bm, WT_SESSION_IMPL *session)
+__bm_compact_start(WT_BM *bm, WT_SESSION_IMPL *session)
{
- return (__wt_block_compact_end(session, bm->block));
+ return (__wt_block_compact_start(session, bm->block));
}
/*
@@ -233,36 +216,25 @@ __bm_is_mapped(WT_BM *bm, WT_SESSION_IMPL *session)
}
/*
- * __bm_stat --
- * Block-manager statistics.
- */
-static int
-__bm_stat(WT_BM *bm, WT_SESSION_IMPL *session, WT_DSRC_STATS *stats)
-{
- __wt_block_stat(session, bm->block, stats);
- return (0);
-}
-
-/*
- * __bm_write --
- * Write a buffer into a block, returning the block's address cookie.
+ * __bm_salvage_end --
+ * End a block manager salvage.
*/
static int
-__bm_write(WT_BM *bm, WT_SESSION_IMPL *session,
- WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_cksum)
+__bm_salvage_end(WT_BM *bm, WT_SESSION_IMPL *session)
{
- return (__wt_block_write(
- session, bm->block, buf, addr, addr_sizep, data_cksum));
+ return (__wt_block_salvage_end(session, bm->block));
}
/*
- * __bm_write_size --
- * Return the buffer size required to write a block.
+ * __bm_salvage_next --
+ * Return the next block from the file.
*/
static int
-__bm_write_size(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep)
+__bm_salvage_next(WT_BM *bm,
+ WT_SESSION_IMPL *session, uint8_t *addr, size_t *addr_sizep, bool *eofp)
{
- return (__wt_block_write_size(session, bm->block, sizep));
+ return (__wt_block_salvage_next(
+ session, bm->block, addr, addr_sizep, eofp));
}
/*
@@ -288,25 +260,47 @@ __bm_salvage_valid(WT_BM *bm,
}
/*
- * __bm_salvage_next --
- * Return the next block from the file.
+ * __bm_stat --
+ * Block-manager statistics.
*/
static int
-__bm_salvage_next(WT_BM *bm,
- WT_SESSION_IMPL *session, uint8_t *addr, size_t *addr_sizep, bool *eofp)
+__bm_stat(WT_BM *bm, WT_SESSION_IMPL *session, WT_DSRC_STATS *stats)
{
- return (__wt_block_salvage_next(
- session, bm->block, addr, addr_sizep, eofp));
+ __wt_block_stat(session, bm->block, stats);
+ return (0);
}
/*
- * __bm_salvage_end --
- * End a block manager salvage.
+ * __bm_sync --
+ * Flush a file to disk.
*/
static int
-__bm_salvage_end(WT_BM *bm, WT_SESSION_IMPL *session)
+__bm_sync(WT_BM *bm, WT_SESSION_IMPL *session, bool async)
{
- return (__wt_block_salvage_end(session, bm->block));
+ return (async ?
+ __wt_fsync_async(session, bm->block->fh) :
+ __wt_fsync(session, bm->block->fh));
+}
+
+/*
+ * __bm_verify_addr --
+ * Verify an address.
+ */
+static int
+__bm_verify_addr(WT_BM *bm,
+ WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size)
+{
+ return (__wt_block_verify_addr(session, bm->block, addr, addr_size));
+}
+
+/*
+ * __bm_verify_end --
+ * End a block manager verify.
+ */
+static int
+__bm_verify_end(WT_BM *bm, WT_SESSION_IMPL *session)
+{
+ return (__wt_block_verify_end(session, bm->block));
}
/*
@@ -321,24 +315,25 @@ __bm_verify_start(WT_BM *bm,
}
/*
- * __bm_verify_addr --
- * Verify an address.
+ * __bm_write --
+ * Write a buffer into a block, returning the block's address cookie.
*/
static int
-__bm_verify_addr(WT_BM *bm,
- WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size)
+__bm_write(WT_BM *bm, WT_SESSION_IMPL *session,
+ WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_cksum)
{
- return (__wt_block_verify_addr(session, bm->block, addr, addr_size));
+ return (__wt_block_write(
+ session, bm->block, buf, addr, addr_sizep, data_cksum));
}
/*
- * __bm_verify_end --
- * End a block manager verify.
+ * __bm_write_size --
+ * Return the buffer size required to write a block.
*/
static int
-__bm_verify_end(WT_BM *bm, WT_SESSION_IMPL *session)
+__bm_write_size(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep)
{
- return (__wt_block_verify_end(session, bm->block));
+ return (__wt_block_write_size(session, bm->block, sizep));
}
/*
diff --git a/src/block/block_open.c b/src/block/block_open.c
index dd0f3f0716a..d9b2f908737 100644
--- a/src/block/block_open.c
+++ b/src/block/block_open.c
@@ -296,15 +296,21 @@ __wt_desc_init(WT_SESSION_IMPL *session, WT_FH *fh, uint32_t allocsize)
WT_RET(__wt_scr_alloc(session, allocsize, &buf));
memset(buf->mem, 0, allocsize);
+ /*
+ * Checksum a little-endian version of the header, and write everything
+ * in little-endian format. The checksum is (potentially) returned in a
+ * big-endian format, swap it into place in a separate step.
+ */
desc = buf->mem;
desc->magic = WT_BLOCK_MAGIC;
desc->majorv = WT_BLOCK_MAJOR_VERSION;
desc->minorv = WT_BLOCK_MINOR_VERSION;
-
- /* Update the checksum. */
desc->cksum = 0;
+ __wt_block_desc_byteswap(desc);
desc->cksum = __wt_cksum(desc, allocsize);
-
+#ifdef WORDS_BIGENDIAN
+ desc->cksum = __wt_bswap32(desc->cksum);
+#endif
ret = __wt_write(session, fh, (wt_off_t)0, (size_t)allocsize, desc);
__wt_scr_free(session, &buf);
@@ -321,7 +327,7 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block)
WT_BLOCK_DESC *desc;
WT_DECL_ITEM(buf);
WT_DECL_RET;
- uint32_t cksum;
+ uint32_t cksum_calculate, cksum_tmp;
/* Use a scratch buffer to get correct alignment for direct I/O. */
WT_RET(__wt_scr_alloc(session, block->allocsize, &buf));
@@ -330,14 +336,19 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block)
WT_ERR(__wt_read(session,
block->fh, (wt_off_t)0, (size_t)block->allocsize, buf->mem));
+ /*
+ * Handle little- and big-endian objects. Objects are written in little-
+ * endian format: save the header checksum, and calculate the checksum
+ * for the header in its little-endian form. Then, restore the header's
+ * checksum, and byte-swap the whole thing as necessary, leaving us with
+ * a calculated checksum that should match the checksum in the header.
+ */
desc = buf->mem;
- WT_ERR(__wt_verbose(session, WT_VERB_BLOCK,
- "%s: magic %" PRIu32
- ", major/minor: %" PRIu32 "/%" PRIu32
- ", checksum %#" PRIx32,
- block->name, desc->magic,
- desc->majorv, desc->minorv,
- desc->cksum));
+ cksum_tmp = desc->cksum;
+ desc->cksum = 0;
+ cksum_calculate = __wt_cksum(desc, block->allocsize);
+ desc->cksum = cksum_tmp;
+ __wt_block_desc_byteswap(desc);
/*
* We fail the open if the checksum fails, or the magic number is wrong
@@ -348,10 +359,7 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block)
* may have entered the wrong file name, and is now frantically pounding
* their interrupt key.
*/
- cksum = desc->cksum;
- desc->cksum = 0;
- if (desc->magic != WT_BLOCK_MAGIC ||
- cksum != __wt_cksum(desc, block->allocsize))
+ if (desc->magic != WT_BLOCK_MAGIC || desc->cksum != cksum_calculate)
WT_ERR_MSG(session, WT_ERROR,
"%s does not appear to be a WiredTiger file", block->name);
@@ -365,6 +373,14 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block)
WT_BLOCK_MAJOR_VERSION, WT_BLOCK_MINOR_VERSION,
desc->majorv, desc->minorv);
+ WT_ERR(__wt_verbose(session, WT_VERB_BLOCK,
+ "%s: magic %" PRIu32
+ ", major/minor: %" PRIu32 "/%" PRIu32
+ ", checksum %#" PRIx32,
+ block->name, desc->magic,
+ desc->majorv, desc->minorv,
+ desc->cksum));
+
err: __wt_scr_free(session, &buf);
return (ret);
}
diff --git a/src/block/block_read.c b/src/block/block_read.c
index 0e5911ecf2a..6e74d7a7793 100644
--- a/src/block/block_read.c
+++ b/src/block/block_read.c
@@ -139,6 +139,7 @@ __wt_block_read_off_blind(
WT_RET(__wt_read(
session, block->fh, offset, (size_t)block->allocsize, buf->mem));
blk = WT_BLOCK_HEADER_REF(buf->mem);
+ __wt_block_header_byteswap(blk);
/*
* Copy out the size and checksum (we're about to re-use the buffer),
@@ -163,7 +164,7 @@ int
__wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
WT_ITEM *buf, wt_off_t offset, uint32_t size, uint32_t cksum)
{
- WT_BLOCK_HEADER *blk;
+ WT_BLOCK_HEADER *blk, swap;
size_t bufsize;
uint32_t page_cksum;
@@ -193,14 +194,26 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
WT_RET(__wt_read(session, block->fh, offset, size, buf->mem));
buf->size = size;
+ /*
+ * We incrementally read through the structure before doing a checksum,
+ * do little- to big-endian handling early on, and then select from the
+ * original or swapped structure as needed.
+ */
blk = WT_BLOCK_HEADER_REF(buf->mem);
- if (blk->cksum == cksum) {
+ __wt_block_header_byteswap_copy(blk, &swap);
+ if (swap.cksum == cksum) {
blk->cksum = 0;
page_cksum = __wt_cksum(buf->mem,
- F_ISSET(blk, WT_BLOCK_DATA_CKSUM) ?
+ F_ISSET(&swap, WT_BLOCK_DATA_CKSUM) ?
size : WT_BLOCK_COMPRESS_SKIP);
- if (page_cksum == cksum)
+ if (page_cksum == cksum) {
+ /*
+ * Swap the page-header as needed; this doesn't belong
+ * here, but it's the best place to catch all callers.
+ */
+ __wt_page_header_byteswap(buf->mem);
return (0);
+ }
if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE))
__wt_errx(session,
@@ -216,7 +229,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
"offset %" PRIuMAX ": block header checksum "
"of %" PRIu32 " doesn't match expected checksum "
"of %" PRIu32,
- size, (uintmax_t)offset, blk->cksum, cksum);
+ size, (uintmax_t)offset, swap.cksum, cksum);
/* Panic if a checksum fails during an ordinary read. */
return (block->verify ||
diff --git a/src/block/block_slvg.c b/src/block/block_slvg.c
index ef22c727db4..a8cccd53023 100644
--- a/src/block/block_slvg.c
+++ b/src/block/block_slvg.c
@@ -126,6 +126,7 @@ __wt_block_salvage_next(WT_SESSION_IMPL *session,
WT_ERR(__wt_read(
session, fh, offset, (size_t)allocsize, tmp->mem));
blk = WT_BLOCK_HEADER_REF(tmp->mem);
+ __wt_block_header_byteswap(blk);
size = blk->disk_size;
cksum = blk->cksum;
diff --git a/src/block/block_write.c b/src/block/block_write.c
index 23f4d7650b9..4c6ac198fe4 100644
--- a/src/block/block_write.c
+++ b/src/block/block_write.c
@@ -203,11 +203,18 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
WT_FH *fh;
size_t align_size;
wt_off_t offset;
+ uint32_t cksum;
bool local_locked;
blk = WT_BLOCK_HEADER_REF(buf->mem);
fh = block->fh;
+ /*
+ * Swap the page-header as needed; this doesn't belong here, but it's
+ * the best place to catch all callers.
+ */
+ __wt_page_header_byteswap(buf->mem);
+
/* Buffers should be aligned for writing. */
if (!F_ISSET(buf, WT_ITEM_ALIGNED)) {
WT_ASSERT(session, F_ISSET(buf, WT_ITEM_ALIGNED));
@@ -255,13 +262,21 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
* because they're not compressed, both to give salvage a quick test
* of whether a block is useful and to give us a test so we don't lose
* the first WT_BLOCK_COMPRESS_SKIP bytes without noticing.
+ *
+ * Checksum a little-endian version of the header, and write everything
+ * in little-endian format. The checksum is (potentially) returned in a
+ * big-endian format, swap it into place in a separate step.
*/
blk->flags = 0;
if (data_cksum)
F_SET(blk, WT_BLOCK_DATA_CKSUM);
blk->cksum = 0;
- blk->cksum = __wt_cksum(
+ __wt_block_header_byteswap(blk);
+ blk->cksum = cksum = __wt_cksum(
buf->mem, data_cksum ? align_size : WT_BLOCK_COMPRESS_SKIP);
+#ifdef WORDS_BIGENDIAN
+ blk->cksum = __wt_bswap32(blk->cksum);
+#endif
/* Pre-allocate some number of extension structures. */
WT_RET(__wt_block_ext_prealloc(session, 5));
@@ -325,11 +340,11 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block,
WT_RET(__wt_verbose(session, WT_VERB_WRITE,
"off %" PRIuMAX ", size %" PRIuMAX ", cksum %" PRIu32,
- (uintmax_t)offset, (uintmax_t)align_size, blk->cksum));
+ (uintmax_t)offset, (uintmax_t)align_size, cksum));
*offsetp = offset;
*sizep = WT_STORE_SIZE(align_size);
- *cksump = blk->cksum;
+ *cksump = cksum;
- return (ret);
+ return (0);
}
diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c
index 7f0f37d95d6..2db3ca7d984 100644
--- a/src/btree/bt_handle.c
+++ b/src/btree/bt_handle.c
@@ -329,7 +329,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
* always inherit from the connection.
*/
WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval));
- if (WT_IS_METADATA(btree->dhandle) || cval.len == 0)
+ if (WT_IS_METADATA(session, btree->dhandle) || cval.len == 0)
btree->kencryptor = conn->kencryptor;
else if (WT_STRING_MATCH("none", cval.str, cval.len))
btree->kencryptor = NULL;
@@ -420,7 +420,7 @@ __wt_btree_tree_open(
* Failure to open metadata means that the database is unavailable.
* Try to provide a helpful failure message.
*/
- if (ret != 0 && WT_IS_METADATA(session->dhandle)) {
+ if (ret != 0 && WT_IS_METADATA(session, session->dhandle)) {
__wt_errx(session,
"WiredTiger has failed to open its metadata");
__wt_errx(session, "This may be due to the database"
diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c
index ac83a21ac6f..bd38451d5d1 100644
--- a/src/btree/bt_split.c
+++ b/src/btree/bt_split.c
@@ -875,16 +875,24 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
/* The split is complete and correct, ignore benign errors. */
complete = WT_ERR_IGNORE;
- WT_ERR(__wt_verbose(session, WT_VERB_SPLIT,
- "%p: %s %s" "split into parent %p, %" PRIu32 " -> %" PRIu32
- " (%s%" PRIu32 ")",
- ref->page, ref->page == NULL ?
- "unknown page type" : __wt_page_type_string(ref->page->type),
- ref->page == NULL ? "reverse " : "", parent,
- parent_entries, result_entries,
- ref->page == NULL ? "-" : "+",
- ref->page == NULL ?
- parent_entries - result_entries : result_entries - parent_entries));
+ /*
+ * !!!
+ * Swapping in the new page index released the page for eviction, we can
+ * no longer look inside the page.
+ */
+
+ if (ref->page == NULL)
+ WT_ERR(__wt_verbose(session, WT_VERB_SPLIT,
+ "%p: reverse split into parent %p, %" PRIu32 " -> %" PRIu32
+ " (-%" PRIu32 ")",
+ ref->page, parent, parent_entries, result_entries,
+ parent_entries - result_entries));
+ else
+ WT_ERR(__wt_verbose(session, WT_VERB_SPLIT,
+ "%p: split into parent %p, %" PRIu32 " -> %" PRIu32
+ " (+%" PRIu32 ")",
+ ref->page, parent, parent_entries, result_entries,
+ result_entries - parent_entries));
/*
* The new page index is in place, free the WT_REF we were splitting and
@@ -935,8 +943,10 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
parent_decr += sizeof(WT_REF);
}
- /* We freed the reference that was split in the loop above. */
- ref = NULL;
+ /*
+ * !!!
+ * The original WT_REF has now been freed, we can no longer look at it.
+ */
/*
* We can't free the previous page index, there may be threads using it.
diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c
index 49a59b89552..d7785c689d9 100644
--- a/src/btree/bt_walk.c
+++ b/src/btree/bt_walk.c
@@ -173,8 +173,8 @@ __page_descend(WT_SESSION_IMPL *session,
WT_PAGE_INDEX *pindex;
/*
- * Ref is a child page into which we're descending, and on which we
- * have a hazard pointer.
+ * We're passed a child page into which we're descending, and on which
+ * we have a hazard pointer.
*/
for (;; __wt_yield()) {
WT_INTL_INDEX_GET(session, page, pindex);
diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c
index c06274cdb17..71564a7b3c5 100644
--- a/src/btree/row_srch.c
+++ b/src/btree/row_srch.c
@@ -287,9 +287,26 @@ restart_page: page = current->page;
WT_INTL_INDEX_GET(session, page, pindex);
- /* Fast-path appends. */
+ /*
+ * Fast-path appends.
+ *
+ * The 0th key on an internal page is a problem for a couple of
+ * reasons. First, we have to force the 0th key to sort less
+ * than any application key, so internal pages don't have to be
+ * updated if the application stores a new, "smallest" key in
+ * the tree. Second, reconciliation is aware of this and will
+ * store a byte of garbage in the 0th key, so the comparison of
+ * an application key and a 0th key is meaningless (but doing
+ * the comparison could still incorrectly modify our tracking
+ * of the leading bytes in each key that we can skip during the
+ * comparison). For these reasons, special-case the 0th key, and
+ * never pass it to a collator.
+ */
if (append_check) {
descent = pindex->index[pindex->entries - 1];
+
+ if (pindex->entries == 1)
+ goto append;
__wt_ref_key(page, descent, &item->data, &item->size);
WT_ERR(__wt_compare(
session, collator, srch_key, item, &cmp));
@@ -307,16 +324,8 @@ restart_page: page = current->page;
* collation order), because doing the tests and error handling
* inside the loop costs about 5%.
*
- * The 0th key on an internal page is a problem for a couple of
- * reasons. First, we have to force the 0th key to sort less
- * than any application key, so internal pages don't have to be
- * updated if the application stores a new, "smallest" key in
- * the tree. Second, reconciliation is aware of this and will
- * store a byte of garbage in the 0th key, so the comparison of
- * an application key and a 0th key is meaningless (but doing
- * the comparison could still incorrectly modify our tracking
- * of the leading bytes in each key that we can skip during the
- * comparison). For these reasons, skip the 0th key.
+ * Reference the comment above about the 0th key: we continue to
+ * special-case it.
*/
base = 1;
limit = pindex->entries - 1;
diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c
index 2f62950a36e..a53f7d96e64 100644
--- a/src/conn/conn_api.c
+++ b/src/conn/conn_api.c
@@ -2065,6 +2065,8 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
* DATABASE HOME, IT'S WHAT WE USE TO DECIDE IF WE'RE CREATING OR NOT.
*/
WT_ERR(__wt_turtle_init(session));
+
+ __wt_metadata_init(session);
WT_ERR(__wt_metadata_cursor(session, NULL));
/* Start the worker threads and run recovery. */
diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c
index dedafc2b102..a3cb26afb27 100644
--- a/src/conn/conn_dhandle.c
+++ b/src/conn/conn_dhandle.c
@@ -421,7 +421,7 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session,
(apply_checkpoints ||
dhandle->checkpoint == NULL) &&
WT_PREFIX_MATCH(dhandle->name, "file:") &&
- !WT_IS_METADATA(dhandle))
+ !WT_IS_METADATA(session, dhandle))
WT_RET(__conn_btree_apply_internal(
session, dhandle, func, cfg));
}
@@ -686,7 +686,7 @@ __wt_conn_dhandle_discard(WT_SESSION_IMPL *session)
*/
restart:
TAILQ_FOREACH(dhandle, &conn->dhqh, q) {
- if (WT_IS_METADATA(dhandle))
+ if (WT_IS_METADATA(session, dhandle))
continue;
WT_WITH_DHANDLE(session, dhandle,
diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c
index ed226393fb0..60f46288072 100644
--- a/src/conn/conn_log.c
+++ b/src/conn/conn_log.c
@@ -180,9 +180,10 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file)
* disk and the checkpoint LSN.
*/
if (backup_file != 0)
- min_lognum = WT_MIN(log->ckpt_lsn.file, backup_file);
+ min_lognum = WT_MIN(log->ckpt_lsn.l.file, backup_file);
else
- min_lognum = WT_MIN(log->ckpt_lsn.file, log->sync_lsn.file);
+ min_lognum = WT_MIN(
+ log->ckpt_lsn.l.file, log->sync_lsn.l.file);
WT_RET(__wt_verbose(session, WT_VERB_LOG,
"log_archive: archive to log number %" PRIu32, min_lognum));
@@ -218,8 +219,7 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file)
* Indicate what is our new earliest LSN. It is the start
* of the log file containing the last checkpoint.
*/
- log->first_lsn.file = min_lognum;
- log->first_lsn.offset = 0;
+ WT_SET_LSN(&log->first_lsn, min_lognum, 0);
if (0)
err: __wt_err(session, ret, "log archive server error");
@@ -317,7 +317,7 @@ __wt_log_truncate_files(
backup_file = 0;
if (cursor != NULL)
backup_file = WT_CURSOR_BACKUP_ID(cursor);
- WT_ASSERT(session, backup_file <= log->alloc_lsn.file);
+ WT_ASSERT(session, backup_file <= log->alloc_lsn.l.file);
WT_RET(__wt_verbose(session, WT_VERB_LOG,
"log_truncate_files: Archive once up to %" PRIu32,
backup_file));
@@ -367,7 +367,7 @@ __log_file_server(void *arg)
* could see mismatched settings. If we do, yield
* until it is set. This should rarely happen.
*/
- while (log->log_close_lsn.file < filenum)
+ while (log->log_close_lsn.l.file < filenum)
__wt_yield();
if (__wt_log_cmp(
@@ -398,10 +398,10 @@ __log_file_server(void *arg)
* actual data and has minimal pre-allocated
* zeroed space.
*/
- WT_ERR(__wt_ftruncate(
- session, close_fh, close_end_lsn.offset));
- close_end_lsn.file++;
- close_end_lsn.offset = 0;
+ WT_ERR(__wt_ftruncate(session,
+ close_fh, close_end_lsn.l.offset));
+ WT_SET_LSN(&close_end_lsn,
+ close_end_lsn.l.file + 1, 0);
__wt_spin_lock(session, &log->log_sync_lock);
locked = true;
WT_ERR(__wt_close(session, &close_fh));
@@ -440,9 +440,9 @@ __log_file_server(void *arg)
* this worker thread process that older file
* immediately.
*/
- if ((log->sync_lsn.file <
- log->bg_sync_lsn.file) ||
- (log->sync_lsn.file < min_lsn.file))
+ if ((log->sync_lsn.l.file <
+ log->bg_sync_lsn.l.file) ||
+ (log->sync_lsn.l.file < min_lsn.l.file))
continue;
WT_ERR(__wt_fsync(session, log->log_fh));
__wt_spin_lock(session, &log->log_sync_lock);
@@ -454,7 +454,8 @@ __log_file_server(void *arg)
if (__wt_log_cmp(
&log->sync_lsn, &min_lsn) <= 0) {
WT_ASSERT(session,
- min_lsn.file == log->sync_lsn.file);
+ min_lsn.l.file ==
+ log->sync_lsn.l.file);
log->sync_lsn = min_lsn;
WT_ERR(__wt_cond_signal(
session, log->log_sync_cond));
@@ -500,9 +501,9 @@ typedef struct {
* Return comparison of a written slot pair by LSN.
*/
#define WT_WRLSN_ENTRY_CMP_LT(entry1, entry2) \
- ((entry1).lsn.file < (entry2).lsn.file || \
- ((entry1).lsn.file == (entry2).lsn.file && \
- (entry1).lsn.offset < (entry2).lsn.offset))
+ ((entry1).lsn.l.file < (entry2).lsn.l.file || \
+ ((entry1).lsn.l.file == (entry2).lsn.l.file && \
+ (entry1).lsn.l.offset < (entry2).lsn.l.offset))
/*
* __wt_log_wrlsn --
@@ -539,7 +540,7 @@ restart:
save_i = i;
slot = &log->slot_pool[i++];
WT_ASSERT(session, slot->slot_state != 0 ||
- slot->slot_release_lsn.file >= log->write_lsn.file);
+ slot->slot_release_lsn.l.file >= log->write_lsn.l.file);
if (slot->slot_state != WT_LOG_SLOT_WRITTEN)
continue;
written[written_i].slot_index = save_i;
@@ -629,9 +630,9 @@ restart:
* the checkpoint LSN is close to the end of
* the record.
*/
- if (slot->slot_start_lsn.offset !=
+ if (slot->slot_start_lsn.l.offset !=
slot->slot_last_offset)
- slot->slot_start_lsn.offset =
+ slot->slot_start_lsn.l.offset =
slot->slot_last_offset;
log->write_start_lsn = slot->slot_start_lsn;
log->write_lsn = slot->slot_end_lsn;
diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c
index a15aabdd6fe..11fe180d7a4 100644
--- a/src/conn/conn_sweep.c
+++ b/src/conn/conn_sweep.c
@@ -26,7 +26,7 @@ __sweep_mark(WT_SESSION_IMPL *session, time_t now)
conn = S2C(session);
TAILQ_FOREACH(dhandle, &conn->dhqh, q) {
- if (WT_IS_METADATA(dhandle))
+ if (WT_IS_METADATA(session, dhandle))
continue;
/*
@@ -124,7 +124,7 @@ __sweep_expire(WT_SESSION_IMPL *session, time_t now)
if (conn->open_btree_count < conn->sweep_handles_min)
break;
- if (WT_IS_METADATA(dhandle) ||
+ if (WT_IS_METADATA(session, dhandle) ||
!F_ISSET(dhandle, WT_DHANDLE_OPEN) ||
dhandle->session_inuse != 0 ||
dhandle->timeofdeath == 0 ||
@@ -230,7 +230,7 @@ __sweep_remove_handles(WT_SESSION_IMPL *session)
dhandle != NULL;
dhandle = dhandle_next) {
dhandle_next = TAILQ_NEXT(dhandle, q);
- if (WT_IS_METADATA(dhandle))
+ if (WT_IS_METADATA(session, dhandle))
continue;
if (!WT_DHANDLE_CAN_DISCARD(dhandle))
continue;
diff --git a/src/cursor/cur_log.c b/src/cursor/cur_log.c
index 35a2d00e6ec..3fcd8a86066 100644
--- a/src/cursor/cur_log.c
+++ b/src/cursor/cur_log.c
@@ -187,13 +187,13 @@ __curlog_kv(WT_SESSION_IMPL *session, WT_CURSOR *cursor)
if (FLD_ISSET(cursor->flags, WT_CURSTD_RAW)) {
memset(&item, 0, sizeof(item));
WT_RET(wiredtiger_struct_size((WT_SESSION *)session,
- &item.size, WT_LOGC_KEY_FORMAT, cl->cur_lsn->file,
- cl->cur_lsn->offset, key_count));
+ &item.size, WT_LOGC_KEY_FORMAT, cl->cur_lsn->l.file,
+ cl->cur_lsn->l.offset, key_count));
WT_RET(__wt_realloc(session, NULL, item.size, &cl->packed_key));
item.data = cl->packed_key;
WT_RET(wiredtiger_struct_pack((WT_SESSION *)session,
cl->packed_key, item.size, WT_LOGC_KEY_FORMAT,
- cl->cur_lsn->file, cl->cur_lsn->offset, key_count));
+ cl->cur_lsn->l.file, cl->cur_lsn->l.offset, key_count));
__wt_cursor_set_key(cursor, &item);
WT_RET(wiredtiger_struct_size((WT_SESSION *)session,
@@ -208,8 +208,8 @@ __curlog_kv(WT_SESSION_IMPL *session, WT_CURSOR *cursor)
cl->opvalue));
__wt_cursor_set_value(cursor, &item);
} else {
- __wt_cursor_set_key(cursor, cl->cur_lsn->file,
- cl->cur_lsn->offset, key_count);
+ __wt_cursor_set_key(cursor, cl->cur_lsn->l.file,
+ cl->cur_lsn->l.offset, key_count);
__wt_cursor_set_value(cursor, cl->txnid, cl->rectype, optype,
fileid, cl->opkey, cl->opvalue);
}
@@ -264,7 +264,7 @@ __curlog_search(WT_CURSOR *cursor)
WT_DECL_RET;
WT_LSN key;
WT_SESSION_IMPL *session;
- uint32_t counter;
+ uint32_t counter, key_file, key_offset;
cl = (WT_CURSOR_LOG *)cursor;
@@ -274,7 +274,8 @@ __curlog_search(WT_CURSOR *cursor)
* !!! We are ignoring the counter and only searching based on the LSN.
*/
WT_ERR(__wt_cursor_get_key((WT_CURSOR *)cl,
- &key.file, &key.offset, &counter));
+ &key_file, &key_offset, &counter));
+ WT_SET_LSN(&key, key_file, key_offset);
ret = __wt_log_scan(session, &key, WT_LOGSCAN_ONE,
__curlog_logrec, cl);
if (ret == ENOENT)
diff --git a/src/docs/build-posix.dox b/src/docs/build-posix.dox
index f61b199bff2..4889bf931c9 100644
--- a/src/docs/build-posix.dox
+++ b/src/docs/build-posix.dox
@@ -116,10 +116,6 @@ The WiredTiger software supports some additional configuration options:
Configure WiredTiger to sleep and wait for a debugger to attach on failure.
<b>DO NOT</b> configure this option in production environments.
-@par \c --enable-bzip2
-Configure WiredTiger for <a href="http://www.bzip.org/">bzip2</a>
-compression; see @ref compression for more information.
-
@par \c --enable-diagnostic
Configure WiredTiger to perform various run-time diagnostic tests.
<b>DO NOT</b> configure this option in production environments.
diff --git a/src/docs/compression.dox b/src/docs/compression.dox
index 56715e20752..0be96835760 100644
--- a/src/docs/compression.dox
+++ b/src/docs/compression.dox
@@ -1,36 +1,7 @@
/*! @m_page{{c,java},compression,Compressors}
This section explains how to configure WiredTiger's builtin support for
-the bzip2, lz4, snappy and zlib compression engines.
-
-@section compression_bzip2 Using bzip2 compression
-
-To use the builtin support for
-<a href="http://www.bzip.org/">Julian Seward's bzip2</a>
-compression, first check that bzip2 is installed in include and library
-directories searched by the compiler. Once bzip2 is installed, you can
-enable bzip2 using the \c --enable-bzip2 option to configure.
-
-If bzip2 is installed in a location not normally searched by the
-compiler toolchain, you'll need to modify the \c CPPFLAGS and \c LDFLAGS
-to indicate these locations. For example, with the bzip2 includes and
-libraries installed in \c /usr/local/include and \c /usr/local/lib, you
-would run configure with the following additional arguments:
-
-@code
---enable-bzip2 CPPFLAGS="-I/usr/local/include" LDFLAGS="-L/usr/local/include"
-@endcode
-
-When opening the WiredTiger database, load the bzip2 shared library as
-an extension. For example, with the WiredTiger library installed in
-\c /usr/local/lib, you would use the following extension:
-
-@snippet ex_all.c Configure bzip2 extension
-
-Finally, when creating the WiredTiger object, set \c block_compressor
-to \c bzip2:
-
-@snippet ex_all.c Create a bzip2 compressed table
+the lz4, snappy and zlib compression engines.
@section compression_lz4 Using LZ4 compression
diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox
index e0239919f0b..e4d85003a1e 100644
--- a/src/docs/upgrading.dox
+++ b/src/docs/upgrading.dox
@@ -27,6 +27,13 @@ reclaimed when the call returns. The performance of this API may differ
from earlier releases.
</dd>
+<dt>Bzip2 compression support</dt>
+<dd>
+Support for the bzip2 compression/decompression engine has been removed
+from the WiredTiger release; remaining compression engines include LZ4,
+snappy and zlib.
+</dd>
+
</dl><hr>
@section version_270 Upgrading to Version 2.7.0
diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox
index 64e25978dd8..1f0d1533ac4 100644
--- a/src/docs/wtperf.dox
+++ b/src/docs/wtperf.dox
@@ -160,7 +160,7 @@ connection configuration string
post-populate compact for LSM merging activity
@par compression (string, default=none)
compression extension. Allowed configuration values are: 'none',
-'bzip', 'lz4', 'snappy', 'zlib'
+'lz4', 'snappy', 'zlib'
@par create (boolean, default=true)
do population phase; false to use existing database
@par database_count (unsigned int, default=1)
diff --git a/src/include/block.h b/src/include/block.h
index 27a140b73a4..10efd35086c 100644
--- a/src/include/block.h
+++ b/src/include/block.h
@@ -289,6 +289,23 @@ struct __wt_block_desc {
#define WT_BLOCK_DESC_SIZE 16
/*
+ * __wt_block_desc_byteswap --
+ * Handle big- and little-endian transformation of a description block.
+ */
+static inline void
+__wt_block_desc_byteswap(WT_BLOCK_DESC *desc)
+{
+#ifdef WORDS_BIGENDIAN
+ desc->magic = __wt_bswap32(desc->magic);
+ desc->majorv = __wt_bswap16(desc->majorv);
+ desc->minorv = __wt_bswap16(desc->minorv);
+ desc->cksum = __wt_bswap32(desc->cksum);
+#else
+ WT_UNUSED(desc);
+#endif
+}
+
+/*
* WT_BLOCK_HEADER --
* Blocks have a common header, a WT_PAGE_HEADER structure followed by a
* block-manager specific structure: WT_BLOCK_HEADER is WiredTiger's default.
@@ -331,6 +348,35 @@ struct __wt_block_header {
#define WT_BLOCK_HEADER_SIZE 12
/*
+ * __wt_block_header_byteswap_copy --
+ * Handle big- and little-endian transformation of a header block,
+ * copying from a source to a target.
+ */
+static inline void
+__wt_block_header_byteswap_copy(WT_BLOCK_HEADER *from, WT_BLOCK_HEADER *to)
+{
+ *to = *from;
+#ifdef WORDS_BIGENDIAN
+ to->disk_size = __wt_bswap32(from->disk_size);
+ to->cksum = __wt_bswap32(from->cksum);
+#endif
+}
+
+/*
+ * __wt_block_header_byteswap --
+ * Handle big- and little-endian transformation of a header block.
+ */
+static inline void
+__wt_block_header_byteswap(WT_BLOCK_HEADER *blk)
+{
+#ifdef WORDS_BIGENDIAN
+ __wt_block_header_byteswap_copy(blk, blk);
+#else
+ WT_UNUSED(blk);
+#endif
+}
+
+/*
* WT_BLOCK_HEADER_BYTE
* WT_BLOCK_HEADER_BYTE_SIZE --
* The first usable data byte on the block (past the combined headers).
diff --git a/src/include/btmem.h b/src/include/btmem.h
index cfbd87f0cae..ee495c52fc8 100644
--- a/src/include/btmem.h
+++ b/src/include/btmem.h
@@ -62,6 +62,23 @@ struct __wt_page_header {
#define WT_PAGE_HEADER_SIZE 28
/*
+ * __wt_page_header_byteswap --
+ * Handle big- and little-endian transformation of a page header.
+ */
+static inline void
+__wt_page_header_byteswap(WT_PAGE_HEADER *dsk)
+{
+#ifdef WORDS_BIGENDIAN
+ dsk->recno = __wt_bswap64(dsk->recno);
+ dsk->write_gen = __wt_bswap64(dsk->write_gen);
+ dsk->mem_size = __wt_bswap32(dsk->mem_size);
+ dsk->u.entries = __wt_bswap32(dsk->u.entries);
+#else
+ WT_UNUSED(dsk);
+#endif
+}
+
+/*
* The block-manager specific information immediately follows the WT_PAGE_HEADER
* structure.
*/
diff --git a/src/include/connection.h b/src/include/connection.h
index 5d61f9456b3..88797e83ad6 100644
--- a/src/include/connection.h
+++ b/src/include/connection.h
@@ -364,6 +364,7 @@ struct __wt_connection_impl {
uint32_t txn_logsync; /* Log sync configuration */
WT_SESSION_IMPL *meta_ckpt_session;/* Metadata checkpoint session */
+ uint64_t meta_uri_hash; /* Metadata file name hash */
WT_SESSION_IMPL *sweep_session; /* Handle sweep session */
wt_thread_t sweep_tid; /* Handle sweep thread */
diff --git a/src/include/extern.h b/src/include/extern.h
index b71f4b12486..1999ff6b732 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -455,6 +455,7 @@ extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_ses
extern int __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value);
extern int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep);
extern void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase);
+extern void __wt_metadata_init(WT_SESSION_IMPL *session);
extern int __wt_metadata_cursor_open( WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp);
extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp);
extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cursorp);
diff --git a/src/include/log.h b/src/include/log.h
index 577f6a888a3..0e676d47b66 100644
--- a/src/include/log.h
+++ b/src/include/log.h
@@ -6,6 +6,23 @@
* See the file LICENSE for redistribution information.
*/
+/*
+ * WT_LSN --
+ * A log sequence number, representing a position in the transaction log.
+ */
+union __wt_lsn {
+ struct {
+#ifdef WORDS_BIGENDIAN
+ uint32_t file;
+ uint32_t offset;
+#else
+ uint32_t offset;
+ uint32_t file;
+#endif
+ } l;
+ uint64_t file_offset;
+};
+
#define WT_LOG_FILENAME "WiredTigerLog" /* Log file name */
#define WT_LOG_PREPNAME "WiredTigerPreplog" /* Log pre-allocated name */
#define WT_LOG_TMPNAME "WiredTigerTmplog" /* Log temporary name */
@@ -13,32 +30,33 @@
/* Logging subsystem declarations. */
#define WT_LOG_ALIGN 128
-#define WT_INIT_LSN(l) do { \
- (l)->file = 1; \
- (l)->offset = 0; \
-} while (0)
+/*
+ * Atomically set the two components of the LSN.
+ */
+#define WT_SET_LSN(l, f, o) (l)->file_offset = (((uint64_t)(f) << 32) + (o))
-#define WT_MAX_LSN(l) do { \
- (l)->file = UINT32_MAX; \
- (l)->offset = INT64_MAX; \
-} while (0)
+#define WT_INIT_LSN(l) WT_SET_LSN((l), 1, 0)
-#define WT_ZERO_LSN(l) do { \
- (l)->file = 0; \
- (l)->offset = 0; \
-} while (0)
+#define WT_MAX_LSN(l) WT_SET_LSN((l), UINT32_MAX, INT32_MAX)
-#define WT_IS_INIT_LSN(l) \
- ((l)->file == 1 && (l)->offset == 0)
-#define WT_IS_MAX_LSN(l) \
- ((l)->file == UINT32_MAX && (l)->offset == INT64_MAX)
+#define WT_ZERO_LSN(l) WT_SET_LSN((l), 0, 0)
+
+/*
+ * Initialize LSN is (1,0). We only need to shift the 1 for comparison.
+ */
+#define WT_IS_INIT_LSN(l) ((l)->file_offset == ((uint64_t)1 << 32))
+/*
+ * XXX Original tested INT32_MAX.
+ */
+#define WT_IS_MAX_LSN(lsn) \
+ ((lsn)->l.file == UINT32_MAX && (lsn)->l.offset == INT32_MAX)
/*
* Both of the macros below need to change if the content of __wt_lsn
* ever changes. The value is the following:
* txnid, record type, operation type, file id, operation key, operation value
*/
-#define WT_LOGC_KEY_FORMAT WT_UNCHECKED_STRING(IqI)
+#define WT_LOGC_KEY_FORMAT WT_UNCHECKED_STRING(III)
#define WT_LOGC_VALUE_FORMAT WT_UNCHECKED_STRING(qIIIuu)
#define WT_LOG_SKIP_HEADER(data) \
@@ -253,6 +271,24 @@ struct __wt_log_record {
};
/*
+ * __wt_log_record_byteswap --
+ * Handle big- and little-endian transformation of the log record
+ * header block.
+ */
+static inline void
+__wt_log_record_byteswap(WT_LOG_RECORD *record)
+{
+#ifdef WORDS_BIGENDIAN
+ record->len = __wt_bswap32(record->len);
+ record->checksum = __wt_bswap32(record->checksum);
+ record->flags = __wt_bswap16(record->flags);
+ record->mem_len = __wt_bswap32(record->mem_len);
+#else
+ WT_UNUSED(record);
+#endif
+}
+
+/*
* WT_LOG_DESC --
* The log file's description.
*/
@@ -267,6 +303,24 @@ struct __wt_log_desc {
};
/*
+ * __wt_log_desc_byteswap --
+ * Handle big- and little-endian transformation of the log file
+ * description block.
+ */
+static inline void
+__wt_log_desc_byteswap(WT_LOG_DESC *desc)
+{
+#ifdef WORDS_BIGENDIAN
+ desc->log_magic = __wt_bswap32(desc->log_magic);
+ desc->majorv = __wt_bswap16(desc->majorv);
+ desc->minorv = __wt_bswap16(desc->minorv);
+ desc->log_size = __wt_bswap64(desc->log_size);
+#else
+ WT_UNUSED(desc);
+#endif
+}
+
+/*
* Flags for __wt_txn_op_printlog.
*/
#define WT_TXN_PRINTLOG_HEX 0x0001 /* Add hex output */
diff --git a/src/include/log.i b/src/include/log.i
index fcdbc72c388..9e6c36291f7 100644
--- a/src/include/log.i
+++ b/src/include/log.i
@@ -16,25 +16,14 @@ static inline int __wt_log_cmp(WT_LSN *lsn1, WT_LSN *lsn2);
static inline int
__wt_log_cmp(WT_LSN *lsn1, WT_LSN *lsn2)
{
- WT_LSN l1, l2;
+ uint64_t l1, l2;
/*
* Read LSNs into local variables so that we only read each field
* once and all comparisons are on the same values.
*/
- l1 = *(volatile WT_LSN *)lsn1;
- l2 = *(volatile WT_LSN *)lsn2;
+ l1 = ((volatile WT_LSN *)lsn1)->file_offset;
+ l2 = ((volatile WT_LSN *)lsn2)->file_offset;
- /*
- * If the file numbers are different we don't need to compare the
- * offset.
- */
- if (l1.file != l2.file)
- return (l1.file < l2.file ? -1 : 1);
- /*
- * If the file numbers are the same, compare the offset.
- */
- if (l1.offset != l2.offset)
- return (l1.offset < l2.offset ? -1 : 1);
- return (0);
+ return (l1 < l2 ? -1 : (l1 > l2 ? 1 : 0));
}
diff --git a/src/include/meta.h b/src/include/meta.h
index e29ec4202dc..d61022c0c44 100644
--- a/src/include/meta.h
+++ b/src/include/meta.h
@@ -30,9 +30,8 @@
* against the metafile URI. The validity is checked on connection open
* when diagnostic is enabled.
*/
-#define WT_METAFILE_NAME_HASH 1045034099109282882LLU /* Metadata file hash */
-#define WT_IS_METADATA(dh) \
- ((dh)->name_hash == WT_METAFILE_NAME_HASH && \
+#define WT_IS_METADATA(session, dh) \
+ ((dh)->name_hash == S2C(session)->meta_uri_hash && \
strcmp((dh)->name, WT_METAFILE_URI) == 0)
#define WT_METAFILE_ID 0 /* Metadata file ID */
diff --git a/src/include/swap.h b/src/include/swap.h
new file mode 100644
index 00000000000..15b7a0d9df9
--- /dev/null
+++ b/src/include/swap.h
@@ -0,0 +1,92 @@
+/*-
+ * Copyright (c) 2014-2016 MongoDB, Inc.
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ * All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1300)
+#include <cstdlib>
+#define __wt_bswap16(v) _byteswap_ushort(v)
+#define __wt_bswap32(v) _byteswap_ulong(v)
+#define __wt_bswap64(v) _byteswap_uint64(v)
+#elif defined(__clang__) && \
+ defined(__clang_major__) && defined(__clang_minor__) && \
+ (__clang_major__ >= 3) && (__clang_minor__ >= 1)
+#if __has_builtin(__builtin_bswap16)
+#define __wt_bswap16(v) __builtin_bswap16(v)
+#endif
+#if __has_builtin(__builtin_bswap32)
+#define __wt_bswap32(v) __builtin_bswap32(v)
+#endif
+#if __has_builtin(__builtin_bswap64)
+#define __wt_bswap64(v) __builtin_bswap64(v)
+#endif
+#elif defined(__GNUC__) && (__GNUC__ >= 4)
+#if __GNUC__ >= 4 && defined(__GNUC_MINOR__) && __GNUC_MINOR__ >= 3
+#define __wt_bswap32(v) __builtin_bswap32(v)
+#define __wt_bswap64(v) __builtin_bswap64(v)
+#endif
+#if __GNUC__ >= 4 && defined(__GNUC_MINOR__) && __GNUC_MINOR__ >= 8
+#define __wt_bswap16(v) __builtin_bswap16(v)
+#endif
+#elif defined(__sun)
+#include <sys/byteorder.h>
+#define __wt_bswap16(v) BSWAP_16(v)
+#define __wt_bswap32(v) BSWAP_32(v)
+#define __wt_bswap64(v) BSWAP_64(v)
+#endif
+
+#if !defined(__wt_bswap64)
+/*
+ * __wt_bswap64 --
+ * 64-bit unsigned little-endian to/from big-endian value.
+ */
+static inline uint64_t
+__wt_bswap64(uint64_t v)
+{
+ return (
+ ((v << 56) & 0xff00000000000000UL) |
+ ((v << 40) & 0x00ff000000000000UL) |
+ ((v << 24) & 0x0000ff0000000000UL) |
+ ((v << 8) & 0x000000ff00000000UL) |
+ ((v >> 8) & 0x00000000ff000000UL) |
+ ((v >> 24) & 0x0000000000ff0000UL) |
+ ((v >> 40) & 0x000000000000ff00UL) |
+ ((v >> 56) & 0x00000000000000ffUL)
+ );
+}
+#endif
+
+#if !defined(__wt_bswap32)
+/*
+ * __wt_bswap32 --
+ * 32-bit unsigned little-endian to/from big-endian value.
+ */
+static inline uint32_t
+__wt_bswap32(uint32_t v)
+{
+ return (
+ ((v << 24) & 0xff000000) |
+ ((v << 8) & 0x00ff0000) |
+ ((v >> 8) & 0x0000ff00) |
+ ((v >> 24) & 0x000000ff)
+ );
+}
+#endif
+
+#if !defined(__wt_bswap16)
+/*
+ * __wt_bswap16 --
+ * 16-bit unsigned little-endian to/from big-endian value.
+ */
+static inline uint16_t
+__wt_bswap16(uint16_t v)
+{
+ return (
+ ((v << 8) & 0xff00) |
+ ((v >> 8) & 0x00ff)
+ );
+}
+#endif
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index 676f95d9b05..767c176b53f 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -72,7 +72,6 @@ struct __wt_event_handler; typedef struct __wt_event_handler WT_EVENT_HANDLER;
struct __wt_extension_api; typedef struct __wt_extension_api WT_EXTENSION_API;
struct __wt_extractor; typedef struct __wt_extractor WT_EXTRACTOR;
struct __wt_item; typedef struct __wt_item WT_ITEM;
-struct __wt_lsn; typedef struct __wt_lsn WT_LSN;
struct __wt_session; typedef struct __wt_session WT_SESSION;
#if defined(SWIGJAVA)
@@ -126,22 +125,6 @@ struct __wt_item {
#endif
};
-/*
- * We rely on this structure being aligned at 64 bits by the compiler,
- * if we were paranoid we could add an unused field to ensure the padding
- * is correct.
- *
- * NOTE: If you change the contents of this structure you must also update
- * the macros in log.h.
- */
-/*!
- * A log sequence number, representing a position in the transaction log.
- */
-struct __wt_lsn {
- uint32_t file; /*!< Log file number */
- wt_off_t offset; /*!< Log file offset */
-};
-
/*!
* The maximum packed size of a 64-bit integer. The ::wiredtiger_struct_pack
* function will pack single long integers into at most this many bytes.
@@ -1006,9 +989,9 @@ struct __wt_session {
* @config{block_compressor, configure a compressor for file blocks.
* Permitted values are \c "none" or custom compression engine name
* created with WT_CONNECTION::add_compressor. If WiredTiger has
- * builtin support for \c "bzip2"\, \c "snappy"\, \c "lz4" or \c "zlib"
- * compression\, these names are also available. See @ref compression
- * for more information., a string; default \c none.}
+ * builtin support for \c "snappy"\, \c "lz4" or \c "zlib" compression\,
+ * these names are also available. See @ref compression for more
+ * information., a string; default \c none.}
* @config{cache_resident, do not ever evict the object's pages from
* cache. Not compatible with LSM tables; see @ref
* tuning_cache_resident for more information., a boolean flag; default
@@ -1839,9 +1822,9 @@ struct __wt_connection {
* @config{&nbsp;&nbsp;&nbsp;&nbsp;compressor, configure a compressor
* for log records. Permitted values are \c "none" or custom
* compression engine name created with WT_CONNECTION::add_compressor.
- * If WiredTiger has builtin support for \c "bzip2"\, \c "snappy"\, \c
- * "lz4" or \c "zlib" compression\, these names are also available. See
- * @ref compression for more information., a string; default \c none.}
+ * If WiredTiger has builtin support for \c "snappy"\, \c "lz4" or \c
+ * "zlib" compression\, these names are also available. See @ref
+ * compression for more information., a string; default \c none.}
* @config{&nbsp;&nbsp;&nbsp;&nbsp;enabled, enable logging subsystem., a
* boolean flag; default \c false.}
* @config{&nbsp;&nbsp;&nbsp;&nbsp;file_max, the maximum size of log
@@ -2308,11 +2291,11 @@ struct __wt_connection {
* @config{&nbsp;&nbsp;&nbsp;&nbsp;compressor, configure a compressor for log
* records. Permitted values are \c "none" or custom compression engine name
* created with WT_CONNECTION::add_compressor. If WiredTiger has builtin
- * support for \c "bzip2"\, \c "snappy"\, \c "lz4" or \c "zlib" compression\,
- * these names are also available. See @ref compression for more information.,
- * a string; default \c none.}
- * @config{&nbsp;&nbsp;&nbsp;&nbsp;enabled, enable
- * logging subsystem., a boolean flag; default \c false.}
+ * support for \c "snappy"\, \c "lz4" or \c "zlib" compression\, these names are
+ * also available. See @ref compression for more information., a string;
+ * default \c none.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;enabled, enable logging
+ * subsystem., a boolean flag; default \c false.}
* @config{&nbsp;&nbsp;&nbsp;&nbsp;file_max, the maximum size of log files., an
* integer between 100KB and 2GB; default \c 100MB.}
* @config{&nbsp;&nbsp;&nbsp;&nbsp;path, the path to a directory into which the
diff --git a/src/include/wt_internal.h b/src/include/wt_internal.h
index 54b5dfd19f4..7ff93b86f92 100644
--- a/src/include/wt_internal.h
+++ b/src/include/wt_internal.h
@@ -284,6 +284,8 @@ struct __wt_txn_state;
typedef struct __wt_txn_state WT_TXN_STATE;
struct __wt_update;
typedef struct __wt_update WT_UPDATE;
+union __wt_lsn;
+ typedef union __wt_lsn WT_LSN;
union __wt_rand_state;
typedef union __wt_rand_state WT_RAND_STATE;
/*
@@ -302,6 +304,7 @@ union __wt_rand_state;
#include "msvc.h"
#endif
#include "hardware.h"
+#include "swap.h"
#include "queue.h"
@@ -324,8 +327,8 @@ union __wt_rand_state;
#include "btmem.h"
#include "btree.h"
#include "cache.h"
-#include "config.h"
#include "compact.h"
+#include "config.h"
#include "cursor.h"
#include "dlh.h"
#include "error.h"
diff --git a/src/log/log.c b/src/log/log.c
index 9424f0f935c..ce2d7191491 100644
--- a/src/log/log.c
+++ b/src/log/log.c
@@ -105,7 +105,7 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn)
* LSN has moved into a later log file and there should be a
* log file ready to close.
*/
- while (log->sync_lsn.file < min_lsn->file) {
+ while (log->sync_lsn.l.file < min_lsn->l.file) {
WT_ERR(__wt_cond_signal(session,
S2C(session)->log_file_cond));
WT_ERR(__wt_cond_wait(session, log->log_sync_cond, 10000));
@@ -116,10 +116,11 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn)
* Sync the directory if the log file entry hasn't been written
* into the directory.
*/
- if (log->sync_dir_lsn.file < min_lsn->file) {
+ if (log->sync_dir_lsn.l.file < min_lsn->l.file) {
WT_ERR(__wt_verbose(session, WT_VERB_LOG,
"log_force_sync: sync directory %s to LSN %d/%lu",
- log->log_dir_fh->name, min_lsn->file, min_lsn->offset));
+ log->log_dir_fh->name,
+ min_lsn->l.file, min_lsn->l.offset));
WT_ERR(__wt_directory_sync_fh(session, log->log_dir_fh));
log->sync_dir_lsn = *min_lsn;
WT_STAT_FAST_CONN_INCR(session, log_sync_dir);
@@ -130,7 +131,7 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn)
if (__wt_log_cmp(&log->sync_lsn, min_lsn) < 0) {
WT_ERR(__wt_verbose(session, WT_VERB_LOG,
"log_force_sync: sync %s to LSN %d/%lu",
- log->log_fh->name, min_lsn->file, min_lsn->offset));
+ log->log_fh->name, min_lsn->l.file, min_lsn->l.offset));
WT_ERR(__wt_fsync(session, log->log_fh));
log->sync_lsn = *min_lsn;
WT_STAT_FAST_CONN_INCR(session, log_sync);
@@ -174,7 +175,7 @@ __wt_log_needs_recovery(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn, bool *recp)
* we can skip recovery.
*/
WT_RET(__wt_curlog_open(session, "log:", NULL, &c));
- c->set_key(c, ckp_lsn->file, ckp_lsn->offset, 0);
+ c->set_key(c, ckp_lsn->l.file, ckp_lsn->l.offset, 0);
if ((ret = c->search(c)) == 0) {
while ((ret = c->next(c)) == 0) {
/*
@@ -278,7 +279,7 @@ __wt_log_get_all_files(WT_SESSION_IMPL *session,
/* Filter out any files that are below the checkpoint LSN. */
for (max = 0, i = 0; i < count; ) {
WT_ERR(__wt_log_extract_lognum(session, files[i], &id));
- if (active_only && id < log->ckpt_lsn.file) {
+ if (active_only && id < log->ckpt_lsn.l.file) {
__wt_free(session, files[i]);
files[i] = files[count - 1];
files[--count] = NULL;
@@ -459,8 +460,8 @@ __log_size_fit(WT_SESSION_IMPL *session, WT_LSN *lsn, uint64_t recsize)
conn = S2C(session);
log = conn->log;
- return (lsn->offset == WT_LOG_FIRST_RECORD ||
- lsn->offset + (wt_off_t)recsize < conn->log_file_max);
+ return (lsn->l.offset == WT_LOG_FIRST_RECORD ||
+ lsn->l.offset + (wt_off_t)recsize < conn->log_file_max);
}
/*
@@ -536,9 +537,7 @@ __log_fill(WT_SESSION_IMPL *session,
WT_MYSLOT *myslot, bool force, WT_ITEM *record, WT_LSN *lsnp)
{
WT_DECL_RET;
- WT_LOG_RECORD *logrec;
- logrec = (WT_LOG_RECORD *)record->mem;
/*
* Call __wt_write or copy into the buffer. For now the offset is the
* real byte offset. If the offset becomes a unit of WT_LOG_ALIGN this
@@ -547,19 +546,19 @@ __log_fill(WT_SESSION_IMPL *session,
*/
if (!force && !F_ISSET(myslot, WT_MYSLOT_UNBUFFERED))
memcpy((char *)myslot->slot->slot_buf.mem + myslot->offset,
- logrec, logrec->len);
+ record->mem, record->size);
else
/*
* If this is a force or unbuffered write, write it now.
*/
WT_ERR(__wt_write(session, myslot->slot->slot_fh,
myslot->offset + myslot->slot->slot_start_offset,
- (size_t)logrec->len, (void *)logrec));
+ record->size, record->mem));
- WT_STAT_FAST_CONN_INCRV(session, log_bytes_written, logrec->len);
+ WT_STAT_FAST_CONN_INCRV(session, log_bytes_written, record->size);
if (lsnp != NULL) {
*lsnp = myslot->slot->slot_start_lsn;
- lsnp->offset += (wt_off_t)myslot->offset;
+ lsnp->l.offset += (uint32_t)myslot->offset;
}
err:
if (ret != 0 && myslot->slot->slot_error == 0)
@@ -596,19 +595,31 @@ __log_file_header(
WT_ASSERT(session, sizeof(WT_LOG_DESC) < log->allocsize);
WT_RET(__wt_scr_alloc(session, log->allocsize, &buf));
memset(buf->mem, 0, log->allocsize);
+ buf->size = log->allocsize;
+
logrec = (WT_LOG_RECORD *)buf->mem;
desc = (WT_LOG_DESC *)logrec->record;
desc->log_magic = WT_LOG_MAGIC;
desc->majorv = WT_LOG_MAJOR_VERSION;
desc->minorv = WT_LOG_MINOR_VERSION;
desc->log_size = (uint64_t)conn->log_file_max;
+ __wt_log_desc_byteswap(desc);
/*
* Now that the record is set up, initialize the record header.
+ *
+ * Checksum a little-endian version of the header, and write everything
+ * in little-endian format. The checksum is (potentially) returned in a
+ * big-endian format, swap it into place in a separate step.
*/
logrec->len = log->allocsize;
logrec->checksum = 0;
+ __wt_log_record_byteswap(logrec);
logrec->checksum = __wt_cksum(logrec, log->allocsize);
+#ifdef WORDS_BIGENDIAN
+ logrec->checksum = __wt_bswap32(logrec->checksum);
+#endif
+
WT_CLEAR(tmp);
memset(&myslot, 0, sizeof(myslot));
myslot.slot = &tmp;
@@ -624,7 +635,7 @@ __log_file_header(
tmp.slot_fh = fh;
} else {
WT_ASSERT(session, fh == NULL);
- WT_ERR(__wt_log_acquire(session, logrec->len, &tmp));
+ WT_ERR(__wt_log_acquire(session, log->allocsize, &tmp));
}
WT_ERR(__log_fill(session, &myslot, true, buf, NULL));
/*
@@ -673,7 +684,9 @@ __log_openfile(WT_SESSION_IMPL *session,
memset(buf->mem, 0, allocsize);
WT_ERR(__wt_read(session, *fh, 0, allocsize, buf->mem));
logrec = (WT_LOG_RECORD *)buf->mem;
+ __wt_log_record_byteswap(logrec);
desc = (WT_LOG_DESC *)logrec->record;
+ __wt_log_desc_byteswap(desc);
if (desc->log_magic != WT_LOG_MAGIC)
WT_PANIC_RET(session, WT_ERROR,
"log file %s corrupted: Bad magic number %" PRIu32,
@@ -829,8 +842,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created)
* We need to setup the LSNs. Set the end LSN and alloc LSN to
* the end of the header.
*/
- log->alloc_lsn.file = log->fileid;
- log->alloc_lsn.offset = WT_LOG_FIRST_RECORD;
+ WT_SET_LSN(&log->alloc_lsn, log->fileid, WT_LOG_FIRST_RECORD);
end_lsn = log->alloc_lsn;
/*
@@ -890,7 +902,7 @@ __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot)
* Pre-allocate on the first real write into the log file, if it
* was just created (i.e. not pre-allocated).
*/
- if (log->alloc_lsn.offset == WT_LOG_FIRST_RECORD && created_log)
+ if (log->alloc_lsn.l.offset == WT_LOG_FIRST_RECORD && created_log)
WT_RET(__log_prealloc(session, log->log_fh));
/*
* Initialize the slot for activation.
@@ -931,8 +943,9 @@ __log_truncate(WT_SESSION_IMPL *session,
/*
* Truncate the log file to the given LSN.
*/
- WT_ERR(__log_openfile(session, false, &log_fh, file_prefix, lsn->file));
- WT_ERR(__wt_ftruncate(session, log_fh, lsn->offset));
+ WT_ERR(__log_openfile(session,
+ false, &log_fh, file_prefix, lsn->l.file));
+ WT_ERR(__wt_ftruncate(session, log_fh, lsn->l.offset));
WT_ERR(__wt_fsync(session, log_fh));
WT_ERR(__wt_close(session, &log_fh));
@@ -946,7 +959,8 @@ __log_truncate(WT_SESSION_IMPL *session,
WT_LOG_FILENAME, &logfiles, &logcount));
for (i = 0; i < logcount; i++) {
WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum));
- if (lognum > lsn->file && lognum < log->trunc_lsn.file) {
+ if (lognum > lsn->l.file &&
+ lognum < log->trunc_lsn.l.file) {
WT_ERR(__log_openfile(session,
false, &log_fh, file_prefix, lognum));
/*
@@ -1111,10 +1125,8 @@ __wt_log_open(WT_SESSION_IMPL *session)
if (firstlog == UINT32_MAX) {
WT_ASSERT(session, logcount == 0);
WT_INIT_LSN(&log->first_lsn);
- } else {
- log->first_lsn.file = firstlog;
- log->first_lsn.offset = 0;
- }
+ } else
+ WT_SET_LSN(&log->first_lsn, firstlog, 0);
/*
* Start logging at the beginning of the next log file, no matter
@@ -1346,7 +1358,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep)
* sync operations. The most recent one will set the LSN to the
* beginning of our file.
*/
- if (log->sync_lsn.file < slot->slot_end_lsn.file ||
+ if (log->sync_lsn.l.file < slot->slot_end_lsn.l.file ||
__wt_spin_trylock(session, &log->log_sync_lock) != 0) {
WT_ERR(__wt_cond_wait(
session, log->log_sync_cond, 10000));
@@ -1366,12 +1378,12 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep)
* now if needed.
*/
if (F_ISSET(slot, WT_SLOT_SYNC_DIR) &&
- (log->sync_dir_lsn.file < sync_lsn.file)) {
+ (log->sync_dir_lsn.l.file < sync_lsn.l.file)) {
WT_ASSERT(session, log->log_dir_fh != NULL);
WT_ERR(__wt_verbose(session, WT_VERB_LOG,
- "log_release: sync directory %s to LSN %d/%lu",
+ "log_release: sync directory %s to LSN %u/%lu",
log->log_dir_fh->name,
- sync_lsn.file, sync_lsn.offset));
+ sync_lsn.l.file, sync_lsn.l.offset));
WT_ERR(__wt_directory_sync_fh(
session, log->log_dir_fh));
log->sync_dir_lsn = sync_lsn;
@@ -1384,8 +1396,9 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep)
if (F_ISSET(slot, WT_SLOT_SYNC) &&
__wt_log_cmp(&log->sync_lsn, &slot->slot_end_lsn) < 0) {
WT_ERR(__wt_verbose(session, WT_VERB_LOG,
- "log_release: sync log %s to LSN %d/%lu",
- log->log_fh->name, sync_lsn.file, sync_lsn.offset));
+ "log_release: sync log %s to LSN %u/%lu",
+ log->log_fh->name,
+ sync_lsn.l.file, sync_lsn.l.offset));
WT_STAT_FAST_CONN_INCR(session, log_sync);
WT_ERR(__wt_fsync(session, log->log_fh));
log->sync_lsn = sync_lsn;
@@ -1426,7 +1439,8 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags,
WT_LOG_RECORD *logrec;
WT_LSN end_lsn, next_lsn, rd_lsn, start_lsn;
wt_off_t log_size;
- uint32_t allocsize, cksum, firstlog, lastlog, lognum, rdup_len, reclen;
+ uint32_t allocsize, firstlog, lastlog, lognum, rdup_len, reclen;
+ uint32_t cksum_calculate, cksum_tmp;
u_int i, logcount;
int firstrecord;
bool eol, partial_record;
@@ -1449,8 +1463,8 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags,
if (LF_ISSET(WT_LOGSCAN_RECOVER))
WT_RET(__wt_verbose(session, WT_VERB_LOG,
- "__wt_log_scan truncating to %u/%" PRIuMAX,
- log->trunc_lsn.file, (uintmax_t)log->trunc_lsn.offset));
+ "__wt_log_scan truncating to %u/%u",
+ log->trunc_lsn.l.file, log->trunc_lsn.l.offset));
if (log != NULL) {
allocsize = log->allocsize;
@@ -1468,8 +1482,8 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags,
"choose either a start LSN or a start flag");
/* Offsets must be on allocation boundaries. */
- if (lsnp->offset % allocsize != 0 ||
- lsnp->file > log->fileid)
+ if (lsnp->l.offset % allocsize != 0 ||
+ lsnp->l.file > log->fileid)
return (WT_NOTFOUND);
/*
@@ -1509,14 +1523,13 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags,
lastlog = WT_MAX(lastlog, lognum);
firstlog = WT_MIN(firstlog, lognum);
}
- start_lsn.file = firstlog;
- end_lsn.file = lastlog;
- start_lsn.offset = end_lsn.offset = 0;
+ WT_SET_LSN(&start_lsn, firstlog, 0);
+ WT_SET_LSN(&end_lsn, lastlog, 0);
__wt_log_files_free(session, logfiles, logcount);
logfiles = NULL;
}
WT_ERR(__log_openfile(
- session, false, &log_fh, WT_LOG_FILENAME, start_lsn.file));
+ session, false, &log_fh, WT_LOG_FILENAME, start_lsn.l.file));
WT_ERR(__wt_filesize(session, log_fh, &log_size));
rd_lsn = start_lsn;
@@ -1524,17 +1537,17 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags,
WT_ERR(__wt_scr_alloc(session, 0, &decryptitem));
WT_ERR(__wt_scr_alloc(session, 0, &uncitem));
for (;;) {
- if (rd_lsn.offset + allocsize > log_size) {
+ if (rd_lsn.l.offset + allocsize > log_size) {
advance:
- if (rd_lsn.offset == log_size)
+ if (rd_lsn.l.offset == log_size)
partial_record = false;
else
/*
* See if there is anything non-zero at the
* end of this log file.
*/
- WT_ERR(__log_has_hole(session,
- log_fh, rd_lsn.offset, &partial_record));
+ WT_ERR(__log_has_hole(session, log_fh,
+ rd_lsn.l.offset, &partial_record));
/*
* If we read the last record, go to the next file.
*/
@@ -1556,16 +1569,15 @@ advance:
*/
if (partial_record)
break;
- rd_lsn.file++;
- rd_lsn.offset = 0;
+ WT_SET_LSN(&rd_lsn, rd_lsn.l.file + 1, 0);
/*
* Avoid an error message when we reach end of log
* by checking here.
*/
- if (rd_lsn.file > end_lsn.file)
+ if (rd_lsn.l.file > end_lsn.l.file)
break;
WT_ERR(__log_openfile(session,
- false, &log_fh, WT_LOG_FILENAME, rd_lsn.file));
+ false, &log_fh, WT_LOG_FILENAME, rd_lsn.l.file));
WT_ERR(__wt_filesize(session, log_fh, &log_size));
eol = false;
continue;
@@ -1575,14 +1587,16 @@ advance:
*/
WT_ASSERT(session, buf->memsize >= allocsize);
WT_ERR(__wt_read(session,
- log_fh, rd_lsn.offset, (size_t)allocsize, buf->mem));
+ log_fh, rd_lsn.l.offset, (size_t)allocsize, buf->mem));
/*
- * First 4 bytes is the real record length. See if we
- * need to read more than the allocation size. We expect
- * that we rarely will have to read more. Most log records
- * will be fairly small.
+ * See if we need to read more than the allocation size. We
+ * expect that we rarely will have to read more. Most log
+ * records will be fairly small.
*/
- reclen = *(uint32_t *)buf->mem;
+ reclen = ((WT_LOG_RECORD *)buf->mem)->len;
+#ifdef WORDS_BIGENDIAN
+ reclen = __wt_bswap32(reclen);
+#endif
/*
* Log files are pre-allocated. We need to detect the
* difference between a hole in the file (where this location
@@ -1596,7 +1610,7 @@ advance:
*/
if (reclen == 0) {
WT_ERR(__log_has_hole(
- session, log_fh, rd_lsn.offset, &eol));
+ session, log_fh, rd_lsn.l.offset, &eol));
if (eol)
/* Found a hole. This LSN is the end. */
break;
@@ -1611,7 +1625,7 @@ advance:
* log record. If we have a partially written record
* then this is considered the end of the log.
*/
- if (rd_lsn.offset + rdup_len > log_size) {
+ if (rd_lsn.l.offset + rdup_len > log_size) {
eol = true;
break;
}
@@ -1620,19 +1634,28 @@ advance:
* record, especially for direct I/O.
*/
WT_ERR(__wt_buf_grow(session, buf, rdup_len));
- WT_ERR(__wt_read(session,
- log_fh, rd_lsn.offset, (size_t)rdup_len, buf->mem));
+ WT_ERR(__wt_read(session, log_fh,
+ rd_lsn.l.offset, (size_t)rdup_len, buf->mem));
WT_STAT_FAST_CONN_INCR(session, log_scan_rereads);
}
/*
* We read in the record, verify checksum.
+ *
+ * Handle little- and big-endian objects. Objects are written
+ * in little-endian format: save the header checksum, and
+ * calculate the checksum for the header in its little-endian
+ * form. Then, restore the header's checksum, and byte-swap
+ * the whole thing as necessary, leaving us with a calculated
+ * checksum that should match the checksum in the header.
*/
buf->size = reclen;
logrec = (WT_LOG_RECORD *)buf->mem;
- cksum = logrec->checksum;
+ cksum_tmp = logrec->checksum;
logrec->checksum = 0;
- logrec->checksum = __wt_cksum(logrec, logrec->len);
- if (logrec->checksum != cksum) {
+ cksum_calculate = __wt_cksum(logrec, reclen);
+ logrec->checksum = cksum_tmp;
+ __wt_log_record_byteswap(logrec);
+ if (logrec->checksum != cksum_calculate) {
/*
* A checksum mismatch means we have reached the end of
* the useful part of the log. This should be found on
@@ -1657,8 +1680,8 @@ advance:
*/
WT_STAT_FAST_CONN_INCR(session, log_scan_records);
next_lsn = rd_lsn;
- next_lsn.offset += (wt_off_t)rdup_len;
- if (rd_lsn.offset != 0) {
+ next_lsn.l.offset += rdup_len;
+ if (rd_lsn.l.offset != 0) {
/*
* We need to manage the different buffers here.
* Buf is the buffer this function uses to read from
@@ -1911,10 +1934,19 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp,
rdup_len - record->size);
record->size = rdup_len;
}
+ /*
+ * Checksum a little-endian version of the header, and write everything
+ * in little-endian format. The checksum is (potentially) returned in a
+ * big-endian format, swap it into place in a separate step.
+ */
logrec = (WT_LOG_RECORD *)record->mem;
logrec->len = (uint32_t)record->size;
logrec->checksum = 0;
+ __wt_log_record_byteswap(logrec);
logrec->checksum = __wt_cksum(logrec, record->size);
+#ifdef WORDS_BIGENDIAN
+ logrec->checksum = __wt_bswap32(logrec->checksum);
+#endif
WT_STAT_FAST_CONN_INCR(session, log_writes);
@@ -2082,7 +2114,8 @@ __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags)
WT_RET(__wt_log_flush_lsn(session, &lsn, false));
WT_RET(__wt_verbose(session, WT_VERB_LOG,
- "log_flush: flags %d LSN %d/%lu", flags, lsn.file, lsn.offset));
+ "log_flush: flags %d LSN %u/%lu",
+ flags, lsn.l.file, lsn.l.offset));
/*
* If the user wants write-no-sync, there is nothing more to do.
* If the user wants background sync, set the LSN and we're done.
diff --git a/src/log/log_slot.c b/src/log/log_slot.c
index 760e8888de6..2844516e78f 100644
--- a/src/log/log_slot.c
+++ b/src/log/log_slot.c
@@ -31,8 +31,8 @@ __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot)
* are reset when the slot is freed. See log_slot_free.
*/
slot->slot_start_lsn = slot->slot_end_lsn = log->alloc_lsn;
- slot->slot_start_offset = log->alloc_lsn.offset;
- slot->slot_last_offset = log->alloc_lsn.offset;
+ slot->slot_start_offset = log->alloc_lsn.l.offset;
+ slot->slot_last_offset = log->alloc_lsn.l.offset;
slot->slot_fh = log->log_fh;
slot->slot_error = 0;
slot->slot_unbuffered = 0;
@@ -96,14 +96,15 @@ retry:
slot->slot_end_lsn = slot->slot_start_lsn;
end_offset =
WT_LOG_SLOT_JOINED_BUFFERED(old_state) + slot->slot_unbuffered;
- slot->slot_end_lsn.offset += (wt_off_t)end_offset;
+ slot->slot_end_lsn.l.offset += end_offset;
WT_STAT_FAST_CONN_INCRV(session,
log_slot_consolidated, end_offset);
/*
* XXX Would like to change so one piece of code advances the LSN.
*/
log->alloc_lsn = slot->slot_end_lsn;
- WT_ASSERT(session, log->alloc_lsn.file >= log->write_lsn.file);
+ WT_ASSERT(session,
+ log->alloc_lsn.l.file >= log->write_lsn.l.file);
return (0);
}
diff --git a/src/meta/meta_ckpt.c b/src/meta/meta_ckpt.c
index f7da8525639..df4cd2cb4d6 100644
--- a/src/meta/meta_ckpt.c
+++ b/src/meta/meta_ckpt.c
@@ -451,7 +451,7 @@ __wt_meta_ckptlist_set(WT_SESSION_IMPL *session,
if (ckptlsn != NULL)
WT_ERR(__wt_buf_catfmt(session, buf,
",checkpoint_lsn=(%" PRIu32 ",%" PRIuMAX ")",
- ckptlsn->file, (uintmax_t)ckptlsn->offset));
+ ckptlsn->l.file, (uintmax_t)ckptlsn->l.offset));
WT_ERR(__ckpt_set(session, fname, buf->mem));
err: __wt_scr_free(session, &buf);
diff --git a/src/meta/meta_table.c b/src/meta/meta_table.c
index 9938cb07a5c..61cc009c983 100644
--- a/src/meta/meta_table.c
+++ b/src/meta/meta_table.c
@@ -9,6 +9,18 @@
#include "wt_internal.h"
/*
+ * __wt_metadata_init --
+ * Metadata initialization.
+ */
+void
+__wt_metadata_init(WT_SESSION_IMPL *session)
+{
+ /* We cache the metadata file's URI hash for fast detection. */
+ S2C(session)->meta_uri_hash =
+ __wt_hash_city64(WT_METAFILE_URI, strlen(WT_METAFILE_URI));
+}
+
+/*
* __metadata_turtle --
* Return if a key's value should be taken from the turtle file.
*/
diff --git a/src/session/session_dhandle.c b/src/session/session_dhandle.c
index 1ac758c0cee..d89bb8d133c 100644
--- a/src/session/session_dhandle.c
+++ b/src/session/session_dhandle.c
@@ -72,7 +72,8 @@ __session_find_dhandle(WT_SESSION_IMPL *session,
bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE;
retry: TAILQ_FOREACH(dhandle_cache, &session->dhhash[bucket], hashq) {
dhandle = dhandle_cache->dhandle;
- if (WT_DHANDLE_INACTIVE(dhandle) && !WT_IS_METADATA(dhandle)) {
+ if (WT_DHANDLE_INACTIVE(dhandle) &&
+ !WT_IS_METADATA(session, dhandle)) {
__session_discard_dhandle(session, dhandle_cache);
/* We deleted our entry, retry from the start. */
goto retry;
@@ -407,7 +408,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session)
difftime(now, dhandle->timeofdeath) >
conn->sweep_idle_time))) {
WT_STAT_FAST_CONN_INCR(session, dh_session_handles);
- WT_ASSERT(session, !WT_IS_METADATA(dhandle));
+ WT_ASSERT(session, !WT_IS_METADATA(session, dhandle));
__session_discard_dhandle(session, dhandle_cache);
}
dhandle_cache = dhandle_cache_next;
diff --git a/src/support/crypto.c b/src/support/crypto.c
index 1049621fb44..ab94ec2c829 100644
--- a/src/support/crypto.c
+++ b/src/support/crypto.c
@@ -21,8 +21,12 @@ __wt_decrypt(WT_SESSION_IMPL *session,
uint32_t encrypt_len;
uint8_t *dst, *src;
- encrypt_len = WT_STORE_SIZE(*((uint32_t *)
- ((uint8_t *)in->data + skip)));
+ encrypt_len =
+ WT_STORE_SIZE(*((uint32_t *)((uint8_t *)in->data + skip)));
+#ifdef WORDS_BIGENDIAN
+ encrypt_len = __wt_bswap32(encrypt_len);
+#endif
+
if (encrypt_len > in->size)
WT_RET_MSG(session, WT_ERROR,
"corrupted encrypted item: padded size less than "
@@ -104,6 +108,9 @@ __wt_encrypt(WT_SESSION_IMPL *session,
* decryption side.
*/
*unpadded_lenp = WT_STORE_SIZE(result_len);
+#ifdef WORDS_BIGENDIAN
+ *unpadded_lenp = __wt_bswap32(*unpadded_lenp);
+#endif
/*
* Copy in the skipped header bytes, set the final data size.
*/
diff --git a/src/support/global.c b/src/support/global.c
index 0234455b6ce..e0d5bafeaa8 100644
--- a/src/support/global.c
+++ b/src/support/global.c
@@ -12,6 +12,35 @@ WT_PROCESS __wt_process; /* Per-process structure */
static int __wt_pthread_once_failed; /* If initialization failed */
/*
+ * __wt_endian_check --
+ * Check the build matches the machine.
+ */
+static int
+__wt_endian_check(void)
+{
+ uint64_t v;
+ bool big;
+ const char *e;
+
+ v = 1;
+ big = *((uint8_t *)&v) == 0;
+
+#ifdef WORDS_BIGENDIAN
+ if (big)
+ return (0);
+ e = "big-endian";
+#else
+ if (!big)
+ return (0);
+ e = "little-endian";
+#endif
+ fprintf(stderr,
+ "This is a %s build of the WiredTiger data engine, incompatible "
+ "with this system\n", e);
+ return (EINVAL);
+}
+
+/*
* __wt_global_once --
* Global initialization, run once.
*/
@@ -31,10 +60,6 @@ __wt_global_once(void)
TAILQ_INIT(&__wt_process.connqh);
#ifdef HAVE_DIAGNOSTIC
- /* Verify the pre-computed metadata hash. */
- WT_ASSERT(NULL, WT_METAFILE_NAME_HASH ==
- __wt_hash_city64(WT_METAFILE_URI, strlen(WT_METAFILE_URI)));
-
/* Load debugging code the compiler might optimize out. */
(void)__wt_breakpoint();
#endif
@@ -50,6 +75,9 @@ __wt_library_init(void)
static bool first = true;
WT_DECL_RET;
+ /* Check the build matches the machine. */
+ WT_RET(__wt_endian_check());
+
/*
* Do per-process initialization once, before anything else, but only
* once. I don't know how heavy-weight the function (pthread_once, in
diff --git a/src/support/hash_city.c b/src/support/hash_city.c
index 5780cd7b459..f12bfc9a70c 100644
--- a/src/support/hash_city.c
+++ b/src/support/hash_city.c
@@ -86,33 +86,60 @@ static uint32_t UNALIGNED_LOAD32(const char *p) {
return (result);
}
-#if !defined(WORDS_BIGENDIAN)
+#ifdef _MSC_VER
-#define uint32_in_expected_order(x) (x)
-#define uint64_in_expected_order(x) (x)
+#include <stdlib.h>
+#define bswap_32(x) _byteswap_ulong(x)
+#define bswap_64(x) _byteswap_uint64(x)
-#else
+#elif defined(__APPLE__)
-#ifdef __APPLE__
-/* Mac OS X / Darwin features */
+// Mac OS X / Darwin features
#include <libkern/OSByteOrder.h>
#define bswap_32(x) OSSwapInt32(x)
#define bswap_64(x) OSSwapInt64(x)
-#elif defined(__sun)
+#elif defined(__sun) || defined(sun)
#include <sys/byteorder.h>
-#define bswap_32 BSWAP_32
-#define bswap_64 BSWAP_64
+#define bswap_32(x) BSWAP_32(x)
+#define bswap_64(x) BSWAP_64(x)
+
+#elif defined(__FreeBSD__)
+
+#include <sys/endian.h>
+#define bswap_32(x) bswap32(x)
+#define bswap_64(x) bswap64(x)
+
+#elif defined(__OpenBSD__)
+
+#include <sys/types.h>
+#define bswap_32(x) swap32(x)
+#define bswap_64(x) swap64(x)
+
+#elif defined(__NetBSD__)
+
+#include <sys/types.h>
+#include <machine/bswap.h>
+#if defined(__BSWAP_RENAME) && !defined(__bswap_32)
+#define bswap_32(x) bswap32(x)
+#define bswap_64(x) bswap64(x)
+#endif
#else
-#include <byteswap.h>
+
+#define bswap_32(x) __wt_bswap32(x)
+#define bswap_64(x) __wt_bswap64(x)
+
#endif
+#ifdef WORDS_BIGENDIAN
#define uint32_in_expected_order(x) (bswap_32(x))
#define uint64_in_expected_order(x) (bswap_64(x))
-
-#endif /* WORDS_BIGENDIAN */
+#else
+#define uint32_in_expected_order(x) (x)
+#define uint64_in_expected_order(x) (x)
+#endif
static uint64_t Fetch64(const char *p) {
return uint64_in_expected_order(UNALIGNED_LOAD64(p));
diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c
index 7d4d4d5c27c..725e46acb0c 100644
--- a/src/txn/txn_ckpt.c
+++ b/src/txn/txn_ckpt.c
@@ -812,7 +812,7 @@ __checkpoint_worker(WT_SESSION_IMPL *session,
* - On connection close when we know there can't be any races.
*/
WT_ASSERT(session, !need_tracking ||
- WT_IS_METADATA(dhandle) || WT_META_TRACKING(session));
+ WT_IS_METADATA(session, dhandle) || WT_META_TRACKING(session));
/*
* Set the checkpoint LSN to the maximum LSN so that if logging is
@@ -1135,7 +1135,7 @@ fake: /*
* recovery and open a checkpoint that isn't yet durable.
*/
if (F_ISSET(conn, WT_CONN_CKPT_SYNC) &&
- (WT_IS_METADATA(dhandle) ||
+ (WT_IS_METADATA(session, dhandle) ||
!F_ISSET(&session->txn, WT_TXN_RUNNING)))
WT_ERR(__wt_checkpoint_sync(session, NULL));
diff --git a/src/txn/txn_log.c b/src/txn/txn_log.c
index 4c4a7fb3132..37a6e0b3711 100644
--- a/src/txn/txn_log.c
+++ b/src/txn/txn_log.c
@@ -266,14 +266,16 @@ __wt_txn_checkpoint_logread(
WT_LSN *ckpt_lsn)
{
WT_ITEM ckpt_snapshot;
+ uint32_t ckpt_file, ckpt_offset;
u_int ckpt_nsnapshot;
- const char *fmt = WT_UNCHECKED_STRING(IQIU);
+ const char *fmt = WT_UNCHECKED_STRING(IIIU);
WT_RET(__wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt,
- &ckpt_lsn->file, &ckpt_lsn->offset,
+ &ckpt_file, &ckpt_offset,
&ckpt_nsnapshot, &ckpt_snapshot));
WT_UNUSED(ckpt_nsnapshot);
WT_UNUSED(ckpt_snapshot);
+ WT_SET_LSN(ckpt_lsn, ckpt_file, ckpt_offset);
*pp = end;
return (0);
}
@@ -294,7 +296,7 @@ __wt_txn_checkpoint_log(
uint8_t *end, *p;
size_t recsize;
uint32_t i, rectype = WT_LOGREC_CHECKPOINT;
- const char *fmt = WT_UNCHECKED_STRING(IIQIU);
+ const char *fmt = WT_UNCHECKED_STRING(IIIIU);
txn = &session->txn;
ckpt_lsn = &txn->ckpt_lsn;
@@ -350,13 +352,13 @@ __wt_txn_checkpoint_log(
/* Write the checkpoint log record. */
WT_ERR(__wt_struct_size(session, &recsize, fmt,
- rectype, ckpt_lsn->file, ckpt_lsn->offset,
+ rectype, ckpt_lsn->l.file, ckpt_lsn->l.offset,
txn->ckpt_nsnapshot, ckpt_snapshot));
WT_ERR(__wt_logrec_alloc(session, recsize, &logrec));
WT_ERR(__wt_struct_pack(session,
(uint8_t *)logrec->data + logrec->size, recsize, fmt,
- rectype, ckpt_lsn->file, ckpt_lsn->offset,
+ rectype, ckpt_lsn->l.file, ckpt_lsn->l.offset,
txn->ckpt_nsnapshot, ckpt_snapshot));
logrec->size += (uint32_t)recsize;
WT_ERR(__wt_log_write(session, logrec, lsnp,
@@ -465,12 +467,11 @@ __txn_printlog(WT_SESSION_IMPL *session,
{
FILE *out;
WT_LOG_RECORD *logrec;
- WT_LSN ckpt_lsn;
WT_TXN_PRINTLOG_ARGS *args;
const uint8_t *end, *p;
const char *msg;
uint64_t txnid;
- uint32_t fileid, rectype;
+ uint32_t fileid, lsnfile, lsnoffset, rectype;
int32_t start;
bool compressed;
@@ -490,8 +491,8 @@ __txn_printlog(WT_SESSION_IMPL *session,
WT_RET(__wt_fprintf(out, ",\n"));
WT_RET(__wt_fprintf(out,
- " { \"lsn\" : [%" PRIu32 ",%" PRId64 "],\n",
- lsnp->file, lsnp->offset));
+ " { \"lsn\" : [%" PRIu32 ",%" PRIu32 "],\n",
+ lsnp->l.file, lsnp->l.offset));
WT_RET(__wt_fprintf(out,
" \"hdr_flags\" : \"%s\",\n", compressed ? "compressed" : ""));
WT_RET(__wt_fprintf(out,
@@ -503,11 +504,11 @@ __txn_printlog(WT_SESSION_IMPL *session,
switch (rectype) {
case WT_LOGREC_CHECKPOINT:
WT_RET(__wt_struct_unpack(session, p, WT_PTRDIFF(end, p),
- WT_UNCHECKED_STRING(IQ), &ckpt_lsn.file, &ckpt_lsn.offset));
+ WT_UNCHECKED_STRING(II), &lsnfile, &lsnoffset));
WT_RET(__wt_fprintf(out, " \"type\" : \"checkpoint\",\n"));
WT_RET(__wt_fprintf(out,
- " \"ckpt_lsn\" : [%" PRIu32 ",%" PRId64 "]\n",
- ckpt_lsn.file, ckpt_lsn.offset));
+ " \"ckpt_lsn\" : [%" PRIu32 ",%" PRIu32 "]\n",
+ lsnfile, lsnoffset));
break;
case WT_LOGREC_COMMIT:
diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c
index 8051d059d7e..e6bd8a8d755 100644
--- a/src/txn/txn_recover.c
+++ b/src/txn/txn_recover.c
@@ -91,9 +91,9 @@ __recovery_cursor(WT_SESSION_IMPL *session, WT_RECOVERY *r,
WT_ERR(__recovery_cursor( \
(session), (r), (lsnp), (fileid), false, (cp))); \
WT_ERR(__wt_verbose((session), WT_VERB_RECOVERY, \
- "%s op %d to file %d at LSN %u/%" PRIuMAX, \
+ "%s op %d to file %d at LSN %u/%u", \
(cursor == NULL) ? "Skipping" : "Applying", \
- optype, fileid, lsnp->file, (uintmax_t)lsnp->offset)); \
+ optype, fileid, lsnp->l.file, lsnp->l.offset)); \
if (cursor == NULL) \
break
@@ -303,8 +303,7 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config)
{
WT_CONFIG_ITEM cval;
WT_LSN lsn;
- intmax_t offset;
- uint32_t fileid;
+ uint32_t fileid, lsnfile, lsnoffset;
WT_RET(__wt_config_getones(r->session, config, "id", &cval));
fileid = (uint32_t)cval.val;
@@ -326,8 +325,8 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config)
if (cval.type != WT_CONFIG_ITEM_STRUCT)
WT_INIT_LSN(&lsn);
else if (sscanf(cval.str,
- "(%" SCNu32 ",%" SCNdMAX ")", &lsn.file, &offset) == 2)
- lsn.offset = offset;
+ "(%" SCNu32 ",%" SCNu32 ")", &lsnfile, &lsnoffset) == 2)
+ WT_SET_LSN(&lsn, lsnfile, lsnoffset);
else
WT_RET_MSG(r->session, EINVAL,
"Failed to parse checkpoint LSN '%.*s'",
@@ -335,8 +334,8 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config)
r->files[fileid].ckpt_lsn = lsn;
WT_RET(__wt_verbose(r->session, WT_VERB_RECOVERY,
- "Recovering %s with id %u @ (%" PRIu32 ", %" PRIu64 ")",
- uri, fileid, lsn.file, lsn.offset));
+ "Recovering %s with id %u @ (%" PRIu32 ", %" PRIu32 ")",
+ uri, fileid, lsn.l.file, lsn.l.offset));
return (0);
@@ -485,8 +484,8 @@ __wt_txn_recover(WT_SESSION_IMPL *session)
*/
r.metadata_only = false;
WT_ERR(__wt_verbose(session, WT_VERB_RECOVERY,
- "Main recovery loop: starting at %u/%" PRIuMAX,
- r.ckpt_lsn.file, (uintmax_t)r.ckpt_lsn.offset));
+ "Main recovery loop: starting at %u/%u",
+ r.ckpt_lsn.l.file, r.ckpt_lsn.l.offset));
WT_ERR(__wt_log_needs_recovery(session, &r.ckpt_lsn, &needs_rec));
/*
* Check if the database was shut down cleanly. If not
diff --git a/test/format/config.c b/test/format/config.c
index 866e210e556..d431546f254 100644
--- a/test/format/config.c
+++ b/test/format/config.c
@@ -244,45 +244,58 @@ config_compression(const char *conf_name)
const char *cstr;
char confbuf[128];
+ /* Return if already specified. */
+ if (config_is_perm(conf_name))
+ return;
+
/*
- * Compression: choose something if compression wasn't specified,
- * otherwise confirm the appropriate shared library is available.
- * We used to verify that the libraries existed but that's no longer
- * robust, since it's possible to build compression libraries into
- * the WiredTiger library.
+ * Don't configure a compression engine for logging if logging isn't
+ * configured (it won't break, but it's confusing).
*/
- if (!config_is_perm(conf_name)) {
- cstr = "none";
- switch (mmrand(NULL, 1, 20)) {
- case 1: case 2: case 3: case 4: /* 20% no compression */
- break;
- case 5: /* 5% bzip */
- cstr = "bzip";
- break;
- case 6: /* 5% bzip-raw */
- cstr = "bzip-raw";
- break;
- case 7: case 8: case 9: case 10: /* 20% lz4 */
- cstr = "lz4";
- break;
- case 11: /* 5% lz4-no-raw */
- cstr = "lz4-noraw";
- break;
- case 12: case 13: case 14: case 15: /* 20% snappy */
- cstr = "snappy";
- break;
- case 16: case 17: case 18: case 19: /* 20% zlib */
- cstr = "zlib";
- break;
- case 20: /* 5% zlib-no-raw */
- cstr = "zlib-noraw";
- break;
- }
-
- (void)snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name,
- cstr);
+ cstr = "none";
+ if (strcmp(conf_name, "logging_compression") == 0 && g.c_logging == 0) {
+ (void)snprintf(
+ confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr);
config_single(confbuf, 0);
+ return;
}
+
+ /*
+ * Select a compression type from the list of built-in engines.
+ *
+ * Listed percentages are only correct if all of the possible engines
+ * are compiled in.
+ */
+ switch (mmrand(NULL, 1, 20)) {
+#ifdef HAVE_BUILTIN_EXTENSION_LZ4
+ case 1: case 2: case 3: case 4: /* 20% lz4 */
+ cstr = "lz4";
+ break;
+ case 5: /* 5% lz4-no-raw */
+ cstr = "lz4-noraw";
+ break;
+#endif
+#ifdef HAVE_BUILTIN_EXTENSION_SNAPPY
+ case 6: case 7: case 8: case 9: /* 30% snappy */
+ case 10: case 11:
+ cstr = "snappy";
+ break;
+#endif
+#ifdef HAVE_BUILTIN_EXTENSION_ZLIB
+ case 12: case 13: case 14: case 15: /* 20% zlib */
+ cstr = "zlib";
+ break;
+ case 16: /* 5% zlib-no-raw */
+ cstr = "zlib-noraw";
+ break;
+#endif
+ case 17: case 18: case 19: case 20: /* 20% no compression */
+ default:
+ break;
+ }
+
+ (void)snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr);
+ config_single(confbuf, 0);
}
/*
@@ -641,10 +654,6 @@ config_map_compression(const char *s, u_int *vp)
{
if (strcmp(s, "none") == 0)
*vp = COMPRESS_NONE;
- else if (strcmp(s, "bzip") == 0)
- *vp = COMPRESS_BZIP;
- else if (strcmp(s, "bzip-raw") == 0)
- *vp = COMPRESS_BZIP_RAW;
else if (strcmp(s, "lz4") == 0)
*vp = COMPRESS_LZ4;
else if (strcmp(s, "lz4-noraw") == 0)
diff --git a/test/format/config.h b/test/format/config.h
index a5190469c7f..d8b11b005d4 100644
--- a/test/format/config.h
+++ b/test/format/config.h
@@ -58,8 +58,7 @@ typedef struct {
} CONFIG;
#define COMPRESSION_LIST \
- "(none | bzip | bzip-raw | lz4 | lz4-noraw | lzo | none | " \
- "snappy | zlib | zlib-noraw)"
+ "(none | lz4 | lz4-noraw | snappy | zlib | zlib-noraw)"
static CONFIG c[] = {
{ "abort",
diff --git a/test/format/format.h b/test/format/format.h
index bf4d1c0a277..41c9de3dd30 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -64,8 +64,6 @@
#define EXTPATH "../../ext/" /* Extensions path */
-#define BZIP_PATH \
- EXTPATH "compressors/bzip2/.libs/libwiredtiger_bzip2.so"
#define LZ4_PATH \
EXTPATH "compressors/lz4/.libs/libwiredtiger_lz4.so"
#define SNAPPY_PATH \
@@ -241,14 +239,12 @@ typedef struct {
u_int c_checksum_flag; /* Checksum flag value */
#define COMPRESS_NONE 1
-#define COMPRESS_BZIP 2
-#define COMPRESS_BZIP_RAW 3
-#define COMPRESS_LZ4 4
-#define COMPRESS_LZ4_NO_RAW 5
-#define COMPRESS_LZO 6
-#define COMPRESS_SNAPPY 7
-#define COMPRESS_ZLIB 8
-#define COMPRESS_ZLIB_NO_RAW 9
+#define COMPRESS_LZ4 2
+#define COMPRESS_LZ4_NO_RAW 3
+#define COMPRESS_LZO 4
+#define COMPRESS_SNAPPY 5
+#define COMPRESS_ZLIB 6
+#define COMPRESS_ZLIB_NO_RAW 7
u_int c_compression_flag; /* Compression flag value */
u_int c_logging_compression_flag; /* Log compression flag value */
diff --git a/test/format/t.c b/test/format/t.c
index 37ba982c987..ccbc0442e4a 100644
--- a/test/format/t.c
+++ b/test/format/t.c
@@ -45,10 +45,14 @@ main(int argc, char *argv[])
config = NULL;
+#ifdef _WIN32
+ g.progname = "t_format.exe";
+#else
if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL)
g.progname = argv[0];
else
++g.progname;
+#endif
#if 0
/* Configure the GNU malloc for debugging. */
diff --git a/test/format/wts.c b/test/format/wts.c
index b75a0c793eb..9d4d3fe5cb8 100644
--- a/test/format/wts.c
+++ b/test/format/wts.c
@@ -38,10 +38,6 @@ compressor(uint32_t compress_flag)
switch (compress_flag) {
case COMPRESS_NONE:
return ("none");
- case COMPRESS_BZIP:
- return ("bzip2");
- case COMPRESS_BZIP_RAW:
- return ("bzip2-raw-test");
case COMPRESS_LZ4:
return ("lz4");
case COMPRESS_LZ4_NO_RAW:
@@ -143,11 +139,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp)
"create,checkpoint_sync=false,cache_size=%" PRIu32 "MB",
g.c_cache);
-#ifdef _WIN32
- p += snprintf(p, REMAIN(p, end), ",error_prefix=\"t_format.exe\"");
-#else
p += snprintf(p, REMAIN(p, end), ",error_prefix=\"%s\"", g.progname);
-#endif
/* In-memory configuration. */
if (g.c_in_memory != 0)
@@ -178,7 +170,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp)
",encryption=(name=%s)", encryptor(g.c_encryption_flag));
/* Miscellaneous. */
-#ifndef _WIN32
+#ifdef HAVE_POSIX_MEMALIGN
p += snprintf(p, REMAIN(p, end), ",buffer_alignment=512");
#endif
@@ -210,9 +202,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp)
/* Extensions. */
p += snprintf(p, REMAIN(p, end),
",extensions=["
- "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],",
+ "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],",
g.c_reverse ? REVERSE_PATH : "",
- access(BZIP_PATH, R_OK) == 0 ? BZIP_PATH : "",
access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "",
access(LZO_PATH, R_OK) == 0 ? LZO_PATH : "",
access(ROTN_PATH, R_OK) == 0 ? ROTN_PATH : "",
diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c
index 637f4752079..4add7a61f66 100644
--- a/test/recovery/truncated-log.c
+++ b/test/recovery/truncated-log.c
@@ -108,7 +108,7 @@ fill_db(void)
* Set to no buffering.
*/
setvbuf(fp, NULL, _IONBF, 0);
- save_lsn.file = 0;
+ save_lsn.l.file = 0;
/*
* Write data into the table until we move to log file 2.
@@ -136,25 +136,25 @@ fill_db(void)
if ((ret = session->open_cursor(
session, "log:", NULL, NULL, &logc)) != 0)
testutil_die(ret, "open_cursor: log");
- if (save_lsn.file != 0) {
- logc->set_key(logc, save_lsn.file,
- save_lsn.offset, 0);
+ if (save_lsn.l.file != 0) {
+ logc->set_key(logc,
+ save_lsn.l.file, save_lsn.l.offset, 0);
if ((ret = logc->search(logc)) != 0)
testutil_die(errno, "search");
}
while ((ret = logc->next(logc)) == 0) {
if ((ret = logc->get_key(logc,
- &lsn.file, &lsn.offset, &unused)) != 0)
+ &lsn.l.file, &lsn.l.offset, &unused)) != 0)
testutil_die(errno, "get_key");
- if (lsn.file < 2)
+ if (lsn.l.file < 2)
save_lsn = lsn;
else {
if (first)
testutil_die(EINVAL,
"min_key too high");
if (fprintf(fp,
- "%" PRIu64 " %" PRIu32 "\n",
- save_lsn.offset, i - 1) == -1)
+ "%" PRIu32 " %" PRIu32 "\n",
+ save_lsn.l.offset, i - 1) == -1)
testutil_die(errno, "fprintf");
fclose(fp);
abort();
diff --git a/test/suite/test_compress01.py b/test/suite/test_compress01.py
index e97953a53cd..94c748fc3e5 100644
--- a/test/suite/test_compress01.py
+++ b/test/suite/test_compress01.py
@@ -42,7 +42,6 @@ class test_compress01(wttest.WiredTigerTestCase):
('table', dict(uri='table:test_compress01')),
]
compress = [
- ('bzip2', dict(compress='bzip2')),
('nop', dict(compress='nop')),
('snappy', dict(compress='snappy')),
('none', dict(compress=None)),
diff --git a/test/suite/test_cursor08.py b/test/suite/test_cursor08.py
index f7bd37a2a3c..1a379518224 100644
--- a/test/suite/test_cursor08.py
+++ b/test/suite/test_cursor08.py
@@ -47,7 +47,6 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess):
('reopen', dict(reopen=True))
])
compress = check_scenarios([
- ('bzip2', dict(compress='bzip2')),
('nop', dict(compress='nop')),
('snappy', dict(compress='snappy')),
('zlib', dict(compress='zlib')),
diff --git a/test/suite/test_encrypt01.py b/test/suite/test_encrypt01.py
index 0521f7c8e8a..0f2782204d2 100644
--- a/test/suite/test_encrypt01.py
+++ b/test/suite/test_encrypt01.py
@@ -55,10 +55,8 @@ class test_encrypt01(wttest.WiredTigerTestCase):
('none', dict(log_compress=None, block_compress=None)),
('nop', dict(log_compress='nop', block_compress='nop')),
('lz4', dict(log_compress='lz4', block_compress='lz4')),
- ('bzip2', dict(log_compress='bzip2', block_compress='bzip2')),
('snappy', dict(log_compress='snappy', block_compress='snappy')),
('zlib', dict(log_compress='zlib', block_compress='zlib')),
- ('bzip2-none', dict(log_compress='bzip2', block_compress=None)),
('none-snappy', dict(log_compress=None, block_compress='snappy')),
('snappy-lz4', dict(log_compress='snappy', block_compress='lz4')),
]
diff --git a/test/suite/test_txn07.py b/test/suite/test_txn07.py
index 94c26990178..f74120e3590 100644
--- a/test/suite/test_txn07.py
+++ b/test/suite/test_txn07.py
@@ -64,7 +64,6 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess):
]
txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))]
compress = [
- ('bzip2', dict(compress='bzip2')),
('nop', dict(compress='nop')),
('snappy', dict(compress='snappy')),
('zlib', dict(compress='zlib')),