diff options
33 files changed, 592 insertions, 206 deletions
diff --git a/dist/s_define.list b/dist/s_define.list index 8b0d9a0bdcd..ff24e3db822 100644 --- a/dist/s_define.list +++ b/dist/s_define.list @@ -55,3 +55,4 @@ WT_WITH_LOCK __F __WIREDTIGER_EXT_H_ __WIREDTIGER_H_ +__wt_bswap16 diff --git a/dist/s_string.ok b/dist/s_string.ok index b2d5a453441..19fa27cd719 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -396,6 +396,7 @@ bool boolean br breakpoint +bswap bt btcur btmem @@ -409,6 +410,7 @@ builtin builtins bytelock bytestring +byteswap byvalue bzCompressEnd bzCompressInit diff --git a/ext/compressors/lz4/lz4_compress.c b/ext/compressors/lz4/lz4_compress.c index d070dc3fb79..3e6609beb80 100644 --- a/ext/compressors/lz4/lz4_compress.c +++ b/ext/compressors/lz4/lz4_compress.c @@ -31,17 +31,7 @@ #include <stdlib.h> #include <string.h> -#include <wiredtiger.h> -#include <wiredtiger_ext.h> - -/* - * We need to include the configuration file to detect whether this extension - * is being built into the WiredTiger library. - */ -#include "wiredtiger_config.h" -#ifdef _MSC_VER -#define inline __inline -#endif +#include <wt_internal.h> /* Local compressor structure. */ typedef struct { @@ -73,6 +63,24 @@ typedef struct { } LZ4_PREFIX; /* + * lz4_prefix_swap -- + * The additional information is written in little-endian format, handle + * the conversion. + */ +static inline void +lz4_prefix_swap(LZ4_PREFIX *prefix) +{ +#ifdef WORDS_BIGENDIAN + prefix->compressed_len = __wt_bswap32(prefix->compressed_len); + prefix->uncompressed_len = __wt_bswap32(prefix->uncompressed_len); + prefix->useful_len = __wt_bswap32(prefix->useful_len); + prefix->unused = __wt_bswap32(prefix->unused); +#else + WT_UNUSED(prefix); +#endif +} + +/* * lz4_error -- * Output an error message, and return a standard error code. */ @@ -119,6 +127,7 @@ lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, prefix.uncompressed_len = (uint32_t)src_len; prefix.useful_len = (uint32_t)src_len; prefix.unused = 0; + lz4_prefix_swap(&prefix); memcpy(dst, &prefix, sizeof(LZ4_PREFIX)); *result_lenp = (size_t)lz4_len + sizeof(LZ4_PREFIX); @@ -154,6 +163,7 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, * decompressed bytes to return from the start of the source buffer. */ memcpy(&prefix, src, sizeof(LZ4_PREFIX)); + lz4_prefix_swap(&prefix); /* * Decompress, starting after the prefix bytes. Use safe decompression: @@ -268,6 +278,7 @@ lz4_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, prefix.uncompressed_len = (uint32_t)sourceSize; prefix.useful_len = offsets[slot]; prefix.unused = 0; + lz4_prefix_swap(&prefix); memcpy(dst, &prefix, sizeof(LZ4_PREFIX)); *result_slotsp = slot; diff --git a/ext/compressors/snappy/snappy_compress.c b/ext/compressors/snappy/snappy_compress.c index b5a347fce81..d1bd2e1813c 100644 --- a/ext/compressors/snappy/snappy_compress.c +++ b/ext/compressors/snappy/snappy_compress.c @@ -31,14 +31,7 @@ #include <stdlib.h> #include <string.h> -#include <wiredtiger.h> -#include <wiredtiger_ext.h> - -/* - * We need to include the configuration file to detect whether this extension - * is being built into the WiredTiger library. - */ -#include "wiredtiger_config.h" +#include <wt_internal.h> /* Local compressor structure. */ typedef struct { @@ -103,16 +96,22 @@ wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, snret = snappy_compress((char *)src, src_len, snapbuf, &snaplen); if (snret == SNAPPY_OK) { - /* - * On decompression, snappy requires the exact compressed byte - * count (the current value of snaplen). WiredTiger does not - * preserve that value, so save snaplen at the beginning of the - * destination buffer. - */ if (snaplen + sizeof(size_t) < src_len) { - *(size_t *)dst = snaplen; *result_lenp = snaplen + sizeof(size_t); *compression_failed = 0; + + /* + * On decompression, snappy requires an exact compressed + * byte count (the current value of snaplen). WiredTiger + * does not preserve that value, so save snaplen at the + * beginning of the destination buffer. + * + * Store the value in little-endian format. + */ +#ifdef WORDS_BIGENDIAN + snaplen = __wt_bswap64(snaplen); +#endif + *(size_t *)dst = snaplen; } else /* The compressor failed to produce a smaller result. */ *compression_failed = 1; @@ -137,8 +136,14 @@ wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, wt_api = ((SNAPPY_COMPRESSOR *)compressor)->wt_api; - /* retrieve the saved length */ + /* + * Retrieve the saved length, handling little- to big-endian conversion + * as necessary. + */ snaplen = *(size_t *)src; +#ifdef WORDS_BIGENDIAN + snaplen = __wt_bswap64(snaplen); +#endif if (snaplen + sizeof(size_t) > src_len) { (void)wt_api->err_printf(wt_api, session, diff --git a/src/block/block_map.c b/src/block/block_map.c index 3d04a492269..b60623a37d8 100644 --- a/src/block/block_map.c +++ b/src/block/block_map.c @@ -20,6 +20,19 @@ __wt_block_map( *(void **)mapp = NULL; *maplenp = 0; +#ifdef WORDS_BIGENDIAN + /* + * The underlying objects are little-endian, mapping objects isn't + * currently supported on big-endian systems. + */ + WT_UNUSED(session); + WT_UNUSED(block); + WT_UNUSED(mappingcookie); +#else + /* Map support is configurable. */ + if (!S2C(session)->mmap) + return (0); + /* * Turn off mapping when verifying the file, because we can't perform * checksum validation of mapped segments, and verify has to checksum @@ -48,6 +61,7 @@ __wt_block_map( * Ignore errors, we'll read the file through the cache if map fails. */ (void)__wt_mmap(session, block->fh, mapp, maplenp, mappingcookie); +#endif return (0); } diff --git a/src/block/block_mgr.c b/src/block/block_mgr.c index 6e2dc775362..dceaae8bb99 100644 --- a/src/block/block_mgr.c +++ b/src/block/block_mgr.c @@ -69,18 +69,6 @@ __bm_checkpoint(WT_BM *bm, } /* - * __bm_sync -- - * Flush a file to disk. - */ -static int -__bm_sync(WT_BM *bm, WT_SESSION_IMPL *session, bool async) -{ - return (async ? - __wt_fsync_async(session, bm->block->fh) : - __wt_fsync(session, bm->block->fh)); -} - -/* * __bm_checkpoint_load -- * Load a checkpoint. */ @@ -89,10 +77,6 @@ __bm_checkpoint_load(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint) { - WT_CONNECTION_IMPL *conn; - - conn = S2C(session); - /* If not opening a checkpoint, we're opening the live system. */ bm->is_live = !checkpoint; WT_RET(__wt_block_checkpoint_load(session, bm->block, @@ -103,9 +87,8 @@ __bm_checkpoint_load(WT_BM *bm, WT_SESSION_IMPL *session, * Read-only objects are optionally mapped into memory instead * of being read into cache buffers. */ - if (conn->mmap) - WT_RET(__wt_block_map(session, bm->block, - &bm->map, &bm->maplen, &bm->mappingcookie)); + WT_RET(__wt_block_map(session, + bm->block, &bm->map, &bm->maplen, &bm->mappingcookie)); /* * If this handle is for a checkpoint, that is, read-only, there @@ -168,13 +151,13 @@ __bm_close(WT_BM *bm, WT_SESSION_IMPL *session) } /* - * __bm_compact_start -- - * Start a block manager compaction. + * __bm_compact_end -- + * End a block manager compaction. */ static int -__bm_compact_start(WT_BM *bm, WT_SESSION_IMPL *session) +__bm_compact_end(WT_BM *bm, WT_SESSION_IMPL *session) { - return (__wt_block_compact_start(session, bm->block)); + return (__wt_block_compact_end(session, bm->block)); } /* @@ -200,13 +183,13 @@ __bm_compact_skip(WT_BM *bm, WT_SESSION_IMPL *session, bool *skipp) } /* - * __bm_compact_end -- - * End a block manager compaction. + * __bm_compact_start -- + * Start a block manager compaction. */ static int -__bm_compact_end(WT_BM *bm, WT_SESSION_IMPL *session) +__bm_compact_start(WT_BM *bm, WT_SESSION_IMPL *session) { - return (__wt_block_compact_end(session, bm->block)); + return (__wt_block_compact_start(session, bm->block)); } /* @@ -233,36 +216,25 @@ __bm_is_mapped(WT_BM *bm, WT_SESSION_IMPL *session) } /* - * __bm_stat -- - * Block-manager statistics. - */ -static int -__bm_stat(WT_BM *bm, WT_SESSION_IMPL *session, WT_DSRC_STATS *stats) -{ - __wt_block_stat(session, bm->block, stats); - return (0); -} - -/* - * __bm_write -- - * Write a buffer into a block, returning the block's address cookie. + * __bm_salvage_end -- + * End a block manager salvage. */ static int -__bm_write(WT_BM *bm, WT_SESSION_IMPL *session, - WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_cksum) +__bm_salvage_end(WT_BM *bm, WT_SESSION_IMPL *session) { - return (__wt_block_write( - session, bm->block, buf, addr, addr_sizep, data_cksum)); + return (__wt_block_salvage_end(session, bm->block)); } /* - * __bm_write_size -- - * Return the buffer size required to write a block. + * __bm_salvage_next -- + * Return the next block from the file. */ static int -__bm_write_size(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep) +__bm_salvage_next(WT_BM *bm, + WT_SESSION_IMPL *session, uint8_t *addr, size_t *addr_sizep, bool *eofp) { - return (__wt_block_write_size(session, bm->block, sizep)); + return (__wt_block_salvage_next( + session, bm->block, addr, addr_sizep, eofp)); } /* @@ -288,25 +260,47 @@ __bm_salvage_valid(WT_BM *bm, } /* - * __bm_salvage_next -- - * Return the next block from the file. + * __bm_stat -- + * Block-manager statistics. */ static int -__bm_salvage_next(WT_BM *bm, - WT_SESSION_IMPL *session, uint8_t *addr, size_t *addr_sizep, bool *eofp) +__bm_stat(WT_BM *bm, WT_SESSION_IMPL *session, WT_DSRC_STATS *stats) { - return (__wt_block_salvage_next( - session, bm->block, addr, addr_sizep, eofp)); + __wt_block_stat(session, bm->block, stats); + return (0); } /* - * __bm_salvage_end -- - * End a block manager salvage. + * __bm_sync -- + * Flush a file to disk. */ static int -__bm_salvage_end(WT_BM *bm, WT_SESSION_IMPL *session) +__bm_sync(WT_BM *bm, WT_SESSION_IMPL *session, bool async) { - return (__wt_block_salvage_end(session, bm->block)); + return (async ? + __wt_fsync_async(session, bm->block->fh) : + __wt_fsync(session, bm->block->fh)); +} + +/* + * __bm_verify_addr -- + * Verify an address. + */ +static int +__bm_verify_addr(WT_BM *bm, + WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) +{ + return (__wt_block_verify_addr(session, bm->block, addr, addr_size)); +} + +/* + * __bm_verify_end -- + * End a block manager verify. + */ +static int +__bm_verify_end(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__wt_block_verify_end(session, bm->block)); } /* @@ -321,24 +315,25 @@ __bm_verify_start(WT_BM *bm, } /* - * __bm_verify_addr -- - * Verify an address. + * __bm_write -- + * Write a buffer into a block, returning the block's address cookie. */ static int -__bm_verify_addr(WT_BM *bm, - WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) +__bm_write(WT_BM *bm, WT_SESSION_IMPL *session, + WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_cksum) { - return (__wt_block_verify_addr(session, bm->block, addr, addr_size)); + return (__wt_block_write( + session, bm->block, buf, addr, addr_sizep, data_cksum)); } /* - * __bm_verify_end -- - * End a block manager verify. + * __bm_write_size -- + * Return the buffer size required to write a block. */ static int -__bm_verify_end(WT_BM *bm, WT_SESSION_IMPL *session) +__bm_write_size(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep) { - return (__wt_block_verify_end(session, bm->block)); + return (__wt_block_write_size(session, bm->block, sizep)); } /* diff --git a/src/block/block_open.c b/src/block/block_open.c index dd0f3f0716a..d9b2f908737 100644 --- a/src/block/block_open.c +++ b/src/block/block_open.c @@ -296,15 +296,21 @@ __wt_desc_init(WT_SESSION_IMPL *session, WT_FH *fh, uint32_t allocsize) WT_RET(__wt_scr_alloc(session, allocsize, &buf)); memset(buf->mem, 0, allocsize); + /* + * Checksum a little-endian version of the header, and write everything + * in little-endian format. The checksum is (potentially) returned in a + * big-endian format, swap it into place in a separate step. + */ desc = buf->mem; desc->magic = WT_BLOCK_MAGIC; desc->majorv = WT_BLOCK_MAJOR_VERSION; desc->minorv = WT_BLOCK_MINOR_VERSION; - - /* Update the checksum. */ desc->cksum = 0; + __wt_block_desc_byteswap(desc); desc->cksum = __wt_cksum(desc, allocsize); - +#ifdef WORDS_BIGENDIAN + desc->cksum = __wt_bswap32(desc->cksum); +#endif ret = __wt_write(session, fh, (wt_off_t)0, (size_t)allocsize, desc); __wt_scr_free(session, &buf); @@ -321,7 +327,7 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_BLOCK_DESC *desc; WT_DECL_ITEM(buf); WT_DECL_RET; - uint32_t cksum; + uint32_t cksum_calculate, cksum_tmp; /* Use a scratch buffer to get correct alignment for direct I/O. */ WT_RET(__wt_scr_alloc(session, block->allocsize, &buf)); @@ -330,14 +336,19 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_ERR(__wt_read(session, block->fh, (wt_off_t)0, (size_t)block->allocsize, buf->mem)); + /* + * Handle little- and big-endian objects. Objects are written in little- + * endian format: save the header checksum, and calculate the checksum + * for the header in its little-endian form. Then, restore the header's + * checksum, and byte-swap the whole thing as necessary, leaving us with + * a calculated checksum that should match the checksum in the header. + */ desc = buf->mem; - WT_ERR(__wt_verbose(session, WT_VERB_BLOCK, - "%s: magic %" PRIu32 - ", major/minor: %" PRIu32 "/%" PRIu32 - ", checksum %#" PRIx32, - block->name, desc->magic, - desc->majorv, desc->minorv, - desc->cksum)); + cksum_tmp = desc->cksum; + desc->cksum = 0; + cksum_calculate = __wt_cksum(desc, block->allocsize); + desc->cksum = cksum_tmp; + __wt_block_desc_byteswap(desc); /* * We fail the open if the checksum fails, or the magic number is wrong @@ -348,10 +359,7 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) * may have entered the wrong file name, and is now frantically pounding * their interrupt key. */ - cksum = desc->cksum; - desc->cksum = 0; - if (desc->magic != WT_BLOCK_MAGIC || - cksum != __wt_cksum(desc, block->allocsize)) + if (desc->magic != WT_BLOCK_MAGIC || desc->cksum != cksum_calculate) WT_ERR_MSG(session, WT_ERROR, "%s does not appear to be a WiredTiger file", block->name); @@ -365,6 +373,14 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_BLOCK_MAJOR_VERSION, WT_BLOCK_MINOR_VERSION, desc->majorv, desc->minorv); + WT_ERR(__wt_verbose(session, WT_VERB_BLOCK, + "%s: magic %" PRIu32 + ", major/minor: %" PRIu32 "/%" PRIu32 + ", checksum %#" PRIx32, + block->name, desc->magic, + desc->majorv, desc->minorv, + desc->cksum)); + err: __wt_scr_free(session, &buf); return (ret); } diff --git a/src/block/block_read.c b/src/block/block_read.c index 0e5911ecf2a..6e74d7a7793 100644 --- a/src/block/block_read.c +++ b/src/block/block_read.c @@ -139,6 +139,7 @@ __wt_block_read_off_blind( WT_RET(__wt_read( session, block->fh, offset, (size_t)block->allocsize, buf->mem)); blk = WT_BLOCK_HEADER_REF(buf->mem); + __wt_block_header_byteswap(blk); /* * Copy out the size and checksum (we're about to re-use the buffer), @@ -163,7 +164,7 @@ int __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t offset, uint32_t size, uint32_t cksum) { - WT_BLOCK_HEADER *blk; + WT_BLOCK_HEADER *blk, swap; size_t bufsize; uint32_t page_cksum; @@ -193,14 +194,26 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_RET(__wt_read(session, block->fh, offset, size, buf->mem)); buf->size = size; + /* + * We incrementally read through the structure before doing a checksum, + * do little- to big-endian handling early on, and then select from the + * original or swapped structure as needed. + */ blk = WT_BLOCK_HEADER_REF(buf->mem); - if (blk->cksum == cksum) { + __wt_block_header_byteswap_copy(blk, &swap); + if (swap.cksum == cksum) { blk->cksum = 0; page_cksum = __wt_cksum(buf->mem, - F_ISSET(blk, WT_BLOCK_DATA_CKSUM) ? + F_ISSET(&swap, WT_BLOCK_DATA_CKSUM) ? size : WT_BLOCK_COMPRESS_SKIP); - if (page_cksum == cksum) + if (page_cksum == cksum) { + /* + * Swap the page-header as needed; this doesn't belong + * here, but it's the best place to catch all callers. + */ + __wt_page_header_byteswap(buf->mem); return (0); + } if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) __wt_errx(session, @@ -216,7 +229,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, "offset %" PRIuMAX ": block header checksum " "of %" PRIu32 " doesn't match expected checksum " "of %" PRIu32, - size, (uintmax_t)offset, blk->cksum, cksum); + size, (uintmax_t)offset, swap.cksum, cksum); /* Panic if a checksum fails during an ordinary read. */ return (block->verify || diff --git a/src/block/block_slvg.c b/src/block/block_slvg.c index ef22c727db4..a8cccd53023 100644 --- a/src/block/block_slvg.c +++ b/src/block/block_slvg.c @@ -126,6 +126,7 @@ __wt_block_salvage_next(WT_SESSION_IMPL *session, WT_ERR(__wt_read( session, fh, offset, (size_t)allocsize, tmp->mem)); blk = WT_BLOCK_HEADER_REF(tmp->mem); + __wt_block_header_byteswap(blk); size = blk->disk_size; cksum = blk->cksum; diff --git a/src/block/block_write.c b/src/block/block_write.c index 23f4d7650b9..4c6ac198fe4 100644 --- a/src/block/block_write.c +++ b/src/block/block_write.c @@ -203,11 +203,18 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_FH *fh; size_t align_size; wt_off_t offset; + uint32_t cksum; bool local_locked; blk = WT_BLOCK_HEADER_REF(buf->mem); fh = block->fh; + /* + * Swap the page-header as needed; this doesn't belong here, but it's + * the best place to catch all callers. + */ + __wt_page_header_byteswap(buf->mem); + /* Buffers should be aligned for writing. */ if (!F_ISSET(buf, WT_ITEM_ALIGNED)) { WT_ASSERT(session, F_ISSET(buf, WT_ITEM_ALIGNED)); @@ -255,13 +262,21 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, * because they're not compressed, both to give salvage a quick test * of whether a block is useful and to give us a test so we don't lose * the first WT_BLOCK_COMPRESS_SKIP bytes without noticing. + * + * Checksum a little-endian version of the header, and write everything + * in little-endian format. The checksum is (potentially) returned in a + * big-endian format, swap it into place in a separate step. */ blk->flags = 0; if (data_cksum) F_SET(blk, WT_BLOCK_DATA_CKSUM); blk->cksum = 0; - blk->cksum = __wt_cksum( + __wt_block_header_byteswap(blk); + blk->cksum = cksum = __wt_cksum( buf->mem, data_cksum ? align_size : WT_BLOCK_COMPRESS_SKIP); +#ifdef WORDS_BIGENDIAN + blk->cksum = __wt_bswap32(blk->cksum); +#endif /* Pre-allocate some number of extension structures. */ WT_RET(__wt_block_ext_prealloc(session, 5)); @@ -325,11 +340,11 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_RET(__wt_verbose(session, WT_VERB_WRITE, "off %" PRIuMAX ", size %" PRIuMAX ", cksum %" PRIu32, - (uintmax_t)offset, (uintmax_t)align_size, blk->cksum)); + (uintmax_t)offset, (uintmax_t)align_size, cksum)); *offsetp = offset; *sizep = WT_STORE_SIZE(align_size); - *cksump = blk->cksum; + *cksump = cksum; - return (ret); + return (0); } diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index 7f0f37d95d6..2db3ca7d984 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -329,7 +329,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) * always inherit from the connection. */ WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval)); - if (WT_IS_METADATA(btree->dhandle) || cval.len == 0) + if (WT_IS_METADATA(session, btree->dhandle) || cval.len == 0) btree->kencryptor = conn->kencryptor; else if (WT_STRING_MATCH("none", cval.str, cval.len)) btree->kencryptor = NULL; @@ -420,7 +420,7 @@ __wt_btree_tree_open( * Failure to open metadata means that the database is unavailable. * Try to provide a helpful failure message. */ - if (ret != 0 && WT_IS_METADATA(session->dhandle)) { + if (ret != 0 && WT_IS_METADATA(session, session->dhandle)) { __wt_errx(session, "WiredTiger has failed to open its metadata"); __wt_errx(session, "This may be due to the database" diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index 49a59b89552..d7785c689d9 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -173,8 +173,8 @@ __page_descend(WT_SESSION_IMPL *session, WT_PAGE_INDEX *pindex; /* - * Ref is a child page into which we're descending, and on which we - * have a hazard pointer. + * We're passed a child page into which we're descending, and on which + * we have a hazard pointer. */ for (;; __wt_yield()) { WT_INTL_INDEX_GET(session, page, pindex); diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 2f62950a36e..a53f7d96e64 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -2065,6 +2065,8 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * DATABASE HOME, IT'S WHAT WE USE TO DECIDE IF WE'RE CREATING OR NOT. */ WT_ERR(__wt_turtle_init(session)); + + __wt_metadata_init(session); WT_ERR(__wt_metadata_cursor(session, NULL)); /* Start the worker threads and run recovery. */ diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index dedafc2b102..a3cb26afb27 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -421,7 +421,7 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, (apply_checkpoints || dhandle->checkpoint == NULL) && WT_PREFIX_MATCH(dhandle->name, "file:") && - !WT_IS_METADATA(dhandle)) + !WT_IS_METADATA(session, dhandle)) WT_RET(__conn_btree_apply_internal( session, dhandle, func, cfg)); } @@ -686,7 +686,7 @@ __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) */ restart: TAILQ_FOREACH(dhandle, &conn->dhqh, q) { - if (WT_IS_METADATA(dhandle)) + if (WT_IS_METADATA(session, dhandle)) continue; WT_WITH_DHANDLE(session, dhandle, diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c index a15aabdd6fe..11fe180d7a4 100644 --- a/src/conn/conn_sweep.c +++ b/src/conn/conn_sweep.c @@ -26,7 +26,7 @@ __sweep_mark(WT_SESSION_IMPL *session, time_t now) conn = S2C(session); TAILQ_FOREACH(dhandle, &conn->dhqh, q) { - if (WT_IS_METADATA(dhandle)) + if (WT_IS_METADATA(session, dhandle)) continue; /* @@ -124,7 +124,7 @@ __sweep_expire(WT_SESSION_IMPL *session, time_t now) if (conn->open_btree_count < conn->sweep_handles_min) break; - if (WT_IS_METADATA(dhandle) || + if (WT_IS_METADATA(session, dhandle) || !F_ISSET(dhandle, WT_DHANDLE_OPEN) || dhandle->session_inuse != 0 || dhandle->timeofdeath == 0 || @@ -230,7 +230,7 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) dhandle != NULL; dhandle = dhandle_next) { dhandle_next = TAILQ_NEXT(dhandle, q); - if (WT_IS_METADATA(dhandle)) + if (WT_IS_METADATA(session, dhandle)) continue; if (!WT_DHANDLE_CAN_DISCARD(dhandle)) continue; diff --git a/src/include/block.h b/src/include/block.h index 27a140b73a4..10efd35086c 100644 --- a/src/include/block.h +++ b/src/include/block.h @@ -289,6 +289,23 @@ struct __wt_block_desc { #define WT_BLOCK_DESC_SIZE 16 /* + * __wt_block_desc_byteswap -- + * Handle big- and little-endian transformation of a description block. + */ +static inline void +__wt_block_desc_byteswap(WT_BLOCK_DESC *desc) +{ +#ifdef WORDS_BIGENDIAN + desc->magic = __wt_bswap32(desc->magic); + desc->majorv = __wt_bswap16(desc->majorv); + desc->minorv = __wt_bswap16(desc->minorv); + desc->cksum = __wt_bswap32(desc->cksum); +#else + WT_UNUSED(desc); +#endif +} + +/* * WT_BLOCK_HEADER -- * Blocks have a common header, a WT_PAGE_HEADER structure followed by a * block-manager specific structure: WT_BLOCK_HEADER is WiredTiger's default. @@ -331,6 +348,35 @@ struct __wt_block_header { #define WT_BLOCK_HEADER_SIZE 12 /* + * __wt_block_header_byteswap_copy -- + * Handle big- and little-endian transformation of a header block, + * copying from a source to a target. + */ +static inline void +__wt_block_header_byteswap_copy(WT_BLOCK_HEADER *from, WT_BLOCK_HEADER *to) +{ + *to = *from; +#ifdef WORDS_BIGENDIAN + to->disk_size = __wt_bswap32(from->disk_size); + to->cksum = __wt_bswap32(from->cksum); +#endif +} + +/* + * __wt_block_header_byteswap -- + * Handle big- and little-endian transformation of a header block. + */ +static inline void +__wt_block_header_byteswap(WT_BLOCK_HEADER *blk) +{ +#ifdef WORDS_BIGENDIAN + __wt_block_header_byteswap_copy(blk, blk); +#else + WT_UNUSED(blk); +#endif +} + +/* * WT_BLOCK_HEADER_BYTE * WT_BLOCK_HEADER_BYTE_SIZE -- * The first usable data byte on the block (past the combined headers). diff --git a/src/include/btmem.h b/src/include/btmem.h index cfbd87f0cae..ee495c52fc8 100644 --- a/src/include/btmem.h +++ b/src/include/btmem.h @@ -62,6 +62,23 @@ struct __wt_page_header { #define WT_PAGE_HEADER_SIZE 28 /* + * __wt_page_header_byteswap -- + * Handle big- and little-endian transformation of a page header. + */ +static inline void +__wt_page_header_byteswap(WT_PAGE_HEADER *dsk) +{ +#ifdef WORDS_BIGENDIAN + dsk->recno = __wt_bswap64(dsk->recno); + dsk->write_gen = __wt_bswap64(dsk->write_gen); + dsk->mem_size = __wt_bswap32(dsk->mem_size); + dsk->u.entries = __wt_bswap32(dsk->u.entries); +#else + WT_UNUSED(dsk); +#endif +} + +/* * The block-manager specific information immediately follows the WT_PAGE_HEADER * structure. */ diff --git a/src/include/connection.h b/src/include/connection.h index 5d61f9456b3..88797e83ad6 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -364,6 +364,7 @@ struct __wt_connection_impl { uint32_t txn_logsync; /* Log sync configuration */ WT_SESSION_IMPL *meta_ckpt_session;/* Metadata checkpoint session */ + uint64_t meta_uri_hash; /* Metadata file name hash */ WT_SESSION_IMPL *sweep_session; /* Handle sweep session */ wt_thread_t sweep_tid; /* Handle sweep thread */ diff --git a/src/include/extern.h b/src/include/extern.h index b71f4b12486..1999ff6b732 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -455,6 +455,7 @@ extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_ses extern int __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value); extern int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep); extern void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase); +extern void __wt_metadata_init(WT_SESSION_IMPL *session); extern int __wt_metadata_cursor_open( WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp); extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp); extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cursorp); diff --git a/src/include/log.h b/src/include/log.h index 6440dca891c..0e676d47b66 100644 --- a/src/include/log.h +++ b/src/include/log.h @@ -271,6 +271,24 @@ struct __wt_log_record { }; /* + * __wt_log_record_byteswap -- + * Handle big- and little-endian transformation of the log record + * header block. + */ +static inline void +__wt_log_record_byteswap(WT_LOG_RECORD *record) +{ +#ifdef WORDS_BIGENDIAN + record->len = __wt_bswap32(record->len); + record->checksum = __wt_bswap32(record->checksum); + record->flags = __wt_bswap16(record->flags); + record->mem_len = __wt_bswap32(record->mem_len); +#else + WT_UNUSED(record); +#endif +} + +/* * WT_LOG_DESC -- * The log file's description. */ @@ -285,6 +303,24 @@ struct __wt_log_desc { }; /* + * __wt_log_desc_byteswap -- + * Handle big- and little-endian transformation of the log file + * description block. + */ +static inline void +__wt_log_desc_byteswap(WT_LOG_DESC *desc) +{ +#ifdef WORDS_BIGENDIAN + desc->log_magic = __wt_bswap32(desc->log_magic); + desc->majorv = __wt_bswap16(desc->majorv); + desc->minorv = __wt_bswap16(desc->minorv); + desc->log_size = __wt_bswap64(desc->log_size); +#else + WT_UNUSED(desc); +#endif +} + +/* * Flags for __wt_txn_op_printlog. */ #define WT_TXN_PRINTLOG_HEX 0x0001 /* Add hex output */ diff --git a/src/include/meta.h b/src/include/meta.h index e29ec4202dc..d61022c0c44 100644 --- a/src/include/meta.h +++ b/src/include/meta.h @@ -30,9 +30,8 @@ * against the metafile URI. The validity is checked on connection open * when diagnostic is enabled. */ -#define WT_METAFILE_NAME_HASH 1045034099109282882LLU /* Metadata file hash */ -#define WT_IS_METADATA(dh) \ - ((dh)->name_hash == WT_METAFILE_NAME_HASH && \ +#define WT_IS_METADATA(session, dh) \ + ((dh)->name_hash == S2C(session)->meta_uri_hash && \ strcmp((dh)->name, WT_METAFILE_URI) == 0) #define WT_METAFILE_ID 0 /* Metadata file ID */ diff --git a/src/include/swap.h b/src/include/swap.h new file mode 100644 index 00000000000..15b7a0d9df9 --- /dev/null +++ b/src/include/swap.h @@ -0,0 +1,92 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#if defined(_MSC_VER) && (_MSC_VER >= 1300) +#include <cstdlib> +#define __wt_bswap16(v) _byteswap_ushort(v) +#define __wt_bswap32(v) _byteswap_ulong(v) +#define __wt_bswap64(v) _byteswap_uint64(v) +#elif defined(__clang__) && \ + defined(__clang_major__) && defined(__clang_minor__) && \ + (__clang_major__ >= 3) && (__clang_minor__ >= 1) +#if __has_builtin(__builtin_bswap16) +#define __wt_bswap16(v) __builtin_bswap16(v) +#endif +#if __has_builtin(__builtin_bswap32) +#define __wt_bswap32(v) __builtin_bswap32(v) +#endif +#if __has_builtin(__builtin_bswap64) +#define __wt_bswap64(v) __builtin_bswap64(v) +#endif +#elif defined(__GNUC__) && (__GNUC__ >= 4) +#if __GNUC__ >= 4 && defined(__GNUC_MINOR__) && __GNUC_MINOR__ >= 3 +#define __wt_bswap32(v) __builtin_bswap32(v) +#define __wt_bswap64(v) __builtin_bswap64(v) +#endif +#if __GNUC__ >= 4 && defined(__GNUC_MINOR__) && __GNUC_MINOR__ >= 8 +#define __wt_bswap16(v) __builtin_bswap16(v) +#endif +#elif defined(__sun) +#include <sys/byteorder.h> +#define __wt_bswap16(v) BSWAP_16(v) +#define __wt_bswap32(v) BSWAP_32(v) +#define __wt_bswap64(v) BSWAP_64(v) +#endif + +#if !defined(__wt_bswap64) +/* + * __wt_bswap64 -- + * 64-bit unsigned little-endian to/from big-endian value. + */ +static inline uint64_t +__wt_bswap64(uint64_t v) +{ + return ( + ((v << 56) & 0xff00000000000000UL) | + ((v << 40) & 0x00ff000000000000UL) | + ((v << 24) & 0x0000ff0000000000UL) | + ((v << 8) & 0x000000ff00000000UL) | + ((v >> 8) & 0x00000000ff000000UL) | + ((v >> 24) & 0x0000000000ff0000UL) | + ((v >> 40) & 0x000000000000ff00UL) | + ((v >> 56) & 0x00000000000000ffUL) + ); +} +#endif + +#if !defined(__wt_bswap32) +/* + * __wt_bswap32 -- + * 32-bit unsigned little-endian to/from big-endian value. + */ +static inline uint32_t +__wt_bswap32(uint32_t v) +{ + return ( + ((v << 24) & 0xff000000) | + ((v << 8) & 0x00ff0000) | + ((v >> 8) & 0x0000ff00) | + ((v >> 24) & 0x000000ff) + ); +} +#endif + +#if !defined(__wt_bswap16) +/* + * __wt_bswap16 -- + * 16-bit unsigned little-endian to/from big-endian value. + */ +static inline uint16_t +__wt_bswap16(uint16_t v) +{ + return ( + ((v << 8) & 0xff00) | + ((v >> 8) & 0x00ff) + ); +} +#endif diff --git a/src/include/wt_internal.h b/src/include/wt_internal.h index 5aed32adfcf..7ff93b86f92 100644 --- a/src/include/wt_internal.h +++ b/src/include/wt_internal.h @@ -304,6 +304,7 @@ union __wt_rand_state; #include "msvc.h" #endif #include "hardware.h" +#include "swap.h" #include "queue.h" @@ -326,8 +327,8 @@ union __wt_rand_state; #include "btmem.h" #include "btree.h" #include "cache.h" -#include "config.h" #include "compact.h" +#include "config.h" #include "cursor.h" #include "dlh.h" #include "error.h" diff --git a/src/log/log.c b/src/log/log.c index fd37b3306e3..6e48bec8bea 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -537,9 +537,7 @@ __log_fill(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool force, WT_ITEM *record, WT_LSN *lsnp) { WT_DECL_RET; - WT_LOG_RECORD *logrec; - logrec = (WT_LOG_RECORD *)record->mem; /* * Call __wt_write or copy into the buffer. For now the offset is the * real byte offset. If the offset becomes a unit of WT_LOG_ALIGN this @@ -548,16 +546,16 @@ __log_fill(WT_SESSION_IMPL *session, */ if (!force && !F_ISSET(myslot, WT_MYSLOT_UNBUFFERED)) memcpy((char *)myslot->slot->slot_buf.mem + myslot->offset, - logrec, logrec->len); + record->mem, record->size); else /* * If this is a force or unbuffered write, write it now. */ WT_ERR(__wt_write(session, myslot->slot->slot_fh, myslot->offset + myslot->slot->slot_start_offset, - (size_t)logrec->len, (void *)logrec)); + record->size, record->mem)); - WT_STAT_FAST_CONN_INCRV(session, log_bytes_written, logrec->len); + WT_STAT_FAST_CONN_INCRV(session, log_bytes_written, record->size); if (lsnp != NULL) { *lsnp = myslot->slot->slot_start_lsn; lsnp->l.offset += (uint32_t)myslot->offset; @@ -597,19 +595,31 @@ __log_file_header( WT_ASSERT(session, sizeof(WT_LOG_DESC) < log->allocsize); WT_RET(__wt_scr_alloc(session, log->allocsize, &buf)); memset(buf->mem, 0, log->allocsize); + buf->size = log->allocsize; + logrec = (WT_LOG_RECORD *)buf->mem; desc = (WT_LOG_DESC *)logrec->record; desc->log_magic = WT_LOG_MAGIC; desc->majorv = WT_LOG_MAJOR_VERSION; desc->minorv = WT_LOG_MINOR_VERSION; desc->log_size = (uint64_t)conn->log_file_max; + __wt_log_desc_byteswap(desc); /* * Now that the record is set up, initialize the record header. + * + * Checksum a little-endian version of the header, and write everything + * in little-endian format. The checksum is (potentially) returned in a + * big-endian format, swap it into place in a separate step. */ logrec->len = log->allocsize; logrec->checksum = 0; + __wt_log_record_byteswap(logrec); logrec->checksum = __wt_cksum(logrec, log->allocsize); +#ifdef WORDS_BIGENDIAN + logrec->checksum = __wt_bswap32(logrec->checksum); +#endif + WT_CLEAR(tmp); memset(&myslot, 0, sizeof(myslot)); myslot.slot = &tmp; @@ -625,7 +635,7 @@ __log_file_header( tmp.slot_fh = fh; } else { WT_ASSERT(session, fh == NULL); - WT_ERR(__wt_log_acquire(session, logrec->len, &tmp)); + WT_ERR(__wt_log_acquire(session, log->allocsize, &tmp)); } WT_ERR(__log_fill(session, &myslot, true, buf, NULL)); /* @@ -674,7 +684,9 @@ __log_openfile(WT_SESSION_IMPL *session, memset(buf->mem, 0, allocsize); WT_ERR(__wt_read(session, *fh, 0, allocsize, buf->mem)); logrec = (WT_LOG_RECORD *)buf->mem; + __wt_log_record_byteswap(logrec); desc = (WT_LOG_DESC *)logrec->record; + __wt_log_desc_byteswap(desc); if (desc->log_magic != WT_LOG_MAGIC) WT_PANIC_RET(session, WT_ERROR, "log file %s corrupted: Bad magic number %" PRIu32, @@ -1427,7 +1439,8 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_LOG_RECORD *logrec; WT_LSN end_lsn, next_lsn, rd_lsn, start_lsn; wt_off_t log_size; - uint32_t allocsize, cksum, firstlog, lastlog, lognum, rdup_len, reclen; + uint32_t allocsize, firstlog, lastlog, lognum, rdup_len, reclen; + uint32_t cksum_calculate, cksum_tmp; u_int i, logcount; int firstrecord; bool eol; @@ -1558,12 +1571,14 @@ advance: WT_ERR(__wt_read(session, log_fh, rd_lsn.l.offset, (size_t)allocsize, buf->mem)); /* - * First 4 bytes is the real record length. See if we - * need to read more than the allocation size. We expect - * that we rarely will have to read more. Most log records - * will be fairly small. + * See if we need to read more than the allocation size. We + * expect that we rarely will have to read more. Most log + * records will be fairly small. */ - reclen = *(uint32_t *)buf->mem; + reclen = ((WT_LOG_RECORD *)buf->mem)->len; +#ifdef WORDS_BIGENDIAN + reclen = __wt_bswap32(reclen); +#endif /* * Log files are pre-allocated. We need to detect the * difference between a hole in the file (where this location @@ -1604,13 +1619,22 @@ advance: } /* * We read in the record, verify checksum. + * + * Handle little- and big-endian objects. Objects are written + * in little-endian format: save the header checksum, and + * calculate the checksum for the header in its little-endian + * form. Then, restore the header's checksum, and byte-swap + * the whole thing as necessary, leaving us with a calculated + * checksum that should match the checksum in the header. */ buf->size = reclen; logrec = (WT_LOG_RECORD *)buf->mem; - cksum = logrec->checksum; + cksum_tmp = logrec->checksum; logrec->checksum = 0; - logrec->checksum = __wt_cksum(logrec, logrec->len); - if (logrec->checksum != cksum) { + cksum_calculate = __wt_cksum(logrec, reclen); + logrec->checksum = cksum_tmp; + __wt_log_record_byteswap(logrec); + if (logrec->checksum != cksum_calculate) { /* * A checksum mismatch means we have reached the end of * the useful part of the log. This should be found on @@ -1889,10 +1913,19 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, rdup_len - record->size); record->size = rdup_len; } + /* + * Checksum a little-endian version of the header, and write everything + * in little-endian format. The checksum is (potentially) returned in a + * big-endian format, swap it into place in a separate step. + */ logrec = (WT_LOG_RECORD *)record->mem; logrec->len = (uint32_t)record->size; logrec->checksum = 0; + __wt_log_record_byteswap(logrec); logrec->checksum = __wt_cksum(logrec, record->size); +#ifdef WORDS_BIGENDIAN + logrec->checksum = __wt_bswap32(logrec->checksum); +#endif WT_STAT_FAST_CONN_INCR(session, log_writes); diff --git a/src/meta/meta_table.c b/src/meta/meta_table.c index 9938cb07a5c..61cc009c983 100644 --- a/src/meta/meta_table.c +++ b/src/meta/meta_table.c @@ -9,6 +9,18 @@ #include "wt_internal.h" /* + * __wt_metadata_init -- + * Metadata initialization. + */ +void +__wt_metadata_init(WT_SESSION_IMPL *session) +{ + /* We cache the metadata file's URI hash for fast detection. */ + S2C(session)->meta_uri_hash = + __wt_hash_city64(WT_METAFILE_URI, strlen(WT_METAFILE_URI)); +} + +/* * __metadata_turtle -- * Return if a key's value should be taken from the turtle file. */ diff --git a/src/session/session_dhandle.c b/src/session/session_dhandle.c index 1ac758c0cee..d89bb8d133c 100644 --- a/src/session/session_dhandle.c +++ b/src/session/session_dhandle.c @@ -72,7 +72,8 @@ __session_find_dhandle(WT_SESSION_IMPL *session, bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; retry: TAILQ_FOREACH(dhandle_cache, &session->dhhash[bucket], hashq) { dhandle = dhandle_cache->dhandle; - if (WT_DHANDLE_INACTIVE(dhandle) && !WT_IS_METADATA(dhandle)) { + if (WT_DHANDLE_INACTIVE(dhandle) && + !WT_IS_METADATA(session, dhandle)) { __session_discard_dhandle(session, dhandle_cache); /* We deleted our entry, retry from the start. */ goto retry; @@ -407,7 +408,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) difftime(now, dhandle->timeofdeath) > conn->sweep_idle_time))) { WT_STAT_FAST_CONN_INCR(session, dh_session_handles); - WT_ASSERT(session, !WT_IS_METADATA(dhandle)); + WT_ASSERT(session, !WT_IS_METADATA(session, dhandle)); __session_discard_dhandle(session, dhandle_cache); } dhandle_cache = dhandle_cache_next; diff --git a/src/support/crypto.c b/src/support/crypto.c index 1049621fb44..ab94ec2c829 100644 --- a/src/support/crypto.c +++ b/src/support/crypto.c @@ -21,8 +21,12 @@ __wt_decrypt(WT_SESSION_IMPL *session, uint32_t encrypt_len; uint8_t *dst, *src; - encrypt_len = WT_STORE_SIZE(*((uint32_t *) - ((uint8_t *)in->data + skip))); + encrypt_len = + WT_STORE_SIZE(*((uint32_t *)((uint8_t *)in->data + skip))); +#ifdef WORDS_BIGENDIAN + encrypt_len = __wt_bswap32(encrypt_len); +#endif + if (encrypt_len > in->size) WT_RET_MSG(session, WT_ERROR, "corrupted encrypted item: padded size less than " @@ -104,6 +108,9 @@ __wt_encrypt(WT_SESSION_IMPL *session, * decryption side. */ *unpadded_lenp = WT_STORE_SIZE(result_len); +#ifdef WORDS_BIGENDIAN + *unpadded_lenp = __wt_bswap32(*unpadded_lenp); +#endif /* * Copy in the skipped header bytes, set the final data size. */ diff --git a/src/support/global.c b/src/support/global.c index 0234455b6ce..e0d5bafeaa8 100644 --- a/src/support/global.c +++ b/src/support/global.c @@ -12,6 +12,35 @@ WT_PROCESS __wt_process; /* Per-process structure */ static int __wt_pthread_once_failed; /* If initialization failed */ /* + * __wt_endian_check -- + * Check the build matches the machine. + */ +static int +__wt_endian_check(void) +{ + uint64_t v; + bool big; + const char *e; + + v = 1; + big = *((uint8_t *)&v) == 0; + +#ifdef WORDS_BIGENDIAN + if (big) + return (0); + e = "big-endian"; +#else + if (!big) + return (0); + e = "little-endian"; +#endif + fprintf(stderr, + "This is a %s build of the WiredTiger data engine, incompatible " + "with this system\n", e); + return (EINVAL); +} + +/* * __wt_global_once -- * Global initialization, run once. */ @@ -31,10 +60,6 @@ __wt_global_once(void) TAILQ_INIT(&__wt_process.connqh); #ifdef HAVE_DIAGNOSTIC - /* Verify the pre-computed metadata hash. */ - WT_ASSERT(NULL, WT_METAFILE_NAME_HASH == - __wt_hash_city64(WT_METAFILE_URI, strlen(WT_METAFILE_URI))); - /* Load debugging code the compiler might optimize out. */ (void)__wt_breakpoint(); #endif @@ -50,6 +75,9 @@ __wt_library_init(void) static bool first = true; WT_DECL_RET; + /* Check the build matches the machine. */ + WT_RET(__wt_endian_check()); + /* * Do per-process initialization once, before anything else, but only * once. I don't know how heavy-weight the function (pthread_once, in diff --git a/src/support/hash_city.c b/src/support/hash_city.c index 5780cd7b459..f12bfc9a70c 100644 --- a/src/support/hash_city.c +++ b/src/support/hash_city.c @@ -86,33 +86,60 @@ static uint32_t UNALIGNED_LOAD32(const char *p) { return (result); } -#if !defined(WORDS_BIGENDIAN) +#ifdef _MSC_VER -#define uint32_in_expected_order(x) (x) -#define uint64_in_expected_order(x) (x) +#include <stdlib.h> +#define bswap_32(x) _byteswap_ulong(x) +#define bswap_64(x) _byteswap_uint64(x) -#else +#elif defined(__APPLE__) -#ifdef __APPLE__ -/* Mac OS X / Darwin features */ +// Mac OS X / Darwin features #include <libkern/OSByteOrder.h> #define bswap_32(x) OSSwapInt32(x) #define bswap_64(x) OSSwapInt64(x) -#elif defined(__sun) +#elif defined(__sun) || defined(sun) #include <sys/byteorder.h> -#define bswap_32 BSWAP_32 -#define bswap_64 BSWAP_64 +#define bswap_32(x) BSWAP_32(x) +#define bswap_64(x) BSWAP_64(x) + +#elif defined(__FreeBSD__) + +#include <sys/endian.h> +#define bswap_32(x) bswap32(x) +#define bswap_64(x) bswap64(x) + +#elif defined(__OpenBSD__) + +#include <sys/types.h> +#define bswap_32(x) swap32(x) +#define bswap_64(x) swap64(x) + +#elif defined(__NetBSD__) + +#include <sys/types.h> +#include <machine/bswap.h> +#if defined(__BSWAP_RENAME) && !defined(__bswap_32) +#define bswap_32(x) bswap32(x) +#define bswap_64(x) bswap64(x) +#endif #else -#include <byteswap.h> + +#define bswap_32(x) __wt_bswap32(x) +#define bswap_64(x) __wt_bswap64(x) + #endif +#ifdef WORDS_BIGENDIAN #define uint32_in_expected_order(x) (bswap_32(x)) #define uint64_in_expected_order(x) (bswap_64(x)) - -#endif /* WORDS_BIGENDIAN */ +#else +#define uint32_in_expected_order(x) (x) +#define uint64_in_expected_order(x) (x) +#endif static uint64_t Fetch64(const char *p) { return uint64_in_expected_order(UNALIGNED_LOAD64(p)); diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 7d4d4d5c27c..725e46acb0c 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -812,7 +812,7 @@ __checkpoint_worker(WT_SESSION_IMPL *session, * - On connection close when we know there can't be any races. */ WT_ASSERT(session, !need_tracking || - WT_IS_METADATA(dhandle) || WT_META_TRACKING(session)); + WT_IS_METADATA(session, dhandle) || WT_META_TRACKING(session)); /* * Set the checkpoint LSN to the maximum LSN so that if logging is @@ -1135,7 +1135,7 @@ fake: /* * recovery and open a checkpoint that isn't yet durable. */ if (F_ISSET(conn, WT_CONN_CKPT_SYNC) && - (WT_IS_METADATA(dhandle) || + (WT_IS_METADATA(session, dhandle) || !F_ISSET(&session->txn, WT_TXN_RUNNING))) WT_ERR(__wt_checkpoint_sync(session, NULL)); diff --git a/test/format/config.c b/test/format/config.c index 0e24527ccea..d431546f254 100644 --- a/test/format/config.c +++ b/test/format/config.c @@ -244,48 +244,58 @@ config_compression(const char *conf_name) const char *cstr; char confbuf[128]; + /* Return if already specified. */ + if (config_is_perm(conf_name)) + return; + /* - * Compression: If compression wasn't specified, select a compression - * type from the list of built-in engines. + * Don't configure a compression engine for logging if logging isn't + * configured (it won't break, but it's confusing). */ - if (!config_is_perm(conf_name)) { - /* - * Listed percentages are only correct if all of the possible - * engines are compiled in. - */ - switch (mmrand(NULL, 1, 20)) { + cstr = "none"; + if (strcmp(conf_name, "logging_compression") == 0 && g.c_logging == 0) { + (void)snprintf( + confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + config_single(confbuf, 0); + return; + } + + /* + * Select a compression type from the list of built-in engines. + * + * Listed percentages are only correct if all of the possible engines + * are compiled in. + */ + switch (mmrand(NULL, 1, 20)) { #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - case 1: case 2: case 3: case 4: /* 20% lz4 */ - cstr = "lz4"; - break; - case 5: /* 5% lz4-no-raw */ - cstr = "lz4-noraw"; - break; + case 1: case 2: case 3: case 4: /* 20% lz4 */ + cstr = "lz4"; + break; + case 5: /* 5% lz4-no-raw */ + cstr = "lz4-noraw"; + break; #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - case 6: case 7: case 8: case 9: /* 30% snappy */ - case 10: case 11: - cstr = "snappy"; - break; + case 6: case 7: case 8: case 9: /* 30% snappy */ + case 10: case 11: + cstr = "snappy"; + break; #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - case 12: case 13: case 14: case 15: /* 20% zlib */ - cstr = "zlib"; - break; - case 16: /* 5% zlib-no-raw */ - cstr = "zlib-noraw"; - break; + case 12: case 13: case 14: case 15: /* 20% zlib */ + cstr = "zlib"; + break; + case 16: /* 5% zlib-no-raw */ + cstr = "zlib-noraw"; + break; #endif - case 17: case 18: case 19: case 20: /* 20% no compression */ - default: - cstr = "none"; - break; - } - - (void)snprintf( - confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); - config_single(confbuf, 0); + case 17: case 18: case 19: case 20: /* 20% no compression */ + default: + break; } + + (void)snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + config_single(confbuf, 0); } /* diff --git a/test/format/t.c b/test/format/t.c index 37ba982c987..ccbc0442e4a 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -45,10 +45,14 @@ main(int argc, char *argv[]) config = NULL; +#ifdef _WIN32 + g.progname = "t_format.exe"; +#else if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) g.progname = argv[0]; else ++g.progname; +#endif #if 0 /* Configure the GNU malloc for debugging. */ diff --git a/test/format/wts.c b/test/format/wts.c index fda92b03efb..9d4d3fe5cb8 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -139,11 +139,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) "create,checkpoint_sync=false,cache_size=%" PRIu32 "MB", g.c_cache); -#ifdef _WIN32 - p += snprintf(p, REMAIN(p, end), ",error_prefix=\"t_format.exe\""); -#else p += snprintf(p, REMAIN(p, end), ",error_prefix=\"%s\"", g.progname); -#endif /* In-memory configuration. */ if (g.c_in_memory != 0) @@ -174,7 +170,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) ",encryption=(name=%s)", encryptor(g.c_encryption_flag)); /* Miscellaneous. */ -#ifndef _WIN32 +#ifdef HAVE_POSIX_MEMALIGN p += snprintf(p, REMAIN(p, end), ",buffer_alignment=512"); #endif |