diff options
author | Matt Kangas <matt.kangas@mongodb.com> | 2014-12-23 08:36:30 -0500 |
---|---|---|
committer | Matt Kangas <matt.kangas@mongodb.com> | 2014-12-23 12:03:28 -0500 |
commit | 55ae19200c7e17249420b4b34799e25b7d1ec92d (patch) | |
tree | 14b03241366fc2d0d53209b4f33371eb64a97d28 /src/third_party/wiredtiger | |
parent | e50c1c5e1e75f2da16f2b8c216b16afe93d83d8d (diff) | |
download | mongo-55ae19200c7e17249420b4b34799e25b7d1ec92d.tar.gz |
Import wiredtiger-wiredtiger-2.8-rc2-311-gbc7ed41.tar.gz from wiredtiger branch mongodb-2.8
Diffstat (limited to 'src/third_party/wiredtiger')
38 files changed, 545 insertions, 259 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 248e967872e..b075afe696a 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -157,7 +157,7 @@ env.Substfile( # # WiredTiger library # -filelistfile = r'dist\filelist.win' +filelistfile = r'build_win\filelist.win' filelist = open(filelistfile) wtsources = [line.strip() for line in filelist @@ -283,7 +283,6 @@ examples = [ "ex_cursor", "ex_data_source", "ex_extending", - "ex_file", "ex_hello", "ex_log", "ex_pack", diff --git a/src/third_party/wiredtiger/build_posix/wiredtiger.pc.in b/src/third_party/wiredtiger/build_posix/wiredtiger.pc.in index be257efcef3..fb45c74bb08 100644 --- a/src/third_party/wiredtiger/build_posix/wiredtiger.pc.in +++ b/src/third_party/wiredtiger/build_posix/wiredtiger.pc.in @@ -7,5 +7,5 @@ Name: WiredTiger Description: The WiredTiger Data Engine Requires: Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lwiredtiger-@VERSION_NOPATCH@ -Cflags: -I${includedir}/wiredtiger-@VERSION_NOPATCH@ +Libs: -L${libdir} -lwiredtiger +Cflags: -I${includedir} diff --git a/src/third_party/wiredtiger/dist/filelist.win b/src/third_party/wiredtiger/build_win/filelist.win index 53e18cbff1a..78ddcb14a59 100644 --- a/src/third_party/wiredtiger/dist/filelist.win +++ b/src/third_party/wiredtiger/build_win/filelist.win @@ -1,6 +1,6 @@ -# filelist -- -# List of source files for WiredTiger Windows library. +# List of source files for WiredTiger library. +# filelist -- src/async/async_api.c src/async/async_op.c src/async/async_worker.c @@ -32,8 +32,8 @@ src/btree/bt_ovfl.c src/btree/bt_page.c src/btree/bt_read.c src/btree/bt_ret.c -src/btree/bt_split.c src/btree/bt_slvg.c +src/btree/bt_split.c src/btree/bt_stat.c src/btree/bt_sync.c src/btree/bt_upgrade.c @@ -113,6 +113,7 @@ src/os_win/os_filesize.c src/os_win/os_flock.c src/os_win/os_fsync.c src/os_win/os_ftruncate.c +src/os_win/os_getenv.c src/os_win/os_map.c src/os_win/os_mtx_cond.c src/os_win/os_once.c @@ -123,6 +124,7 @@ src/os_win/os_remove.c src/os_win/os_rename.c src/os_win/os_rw.c src/os_win/os_sleep.c +src/os_win/os_snprintf.c src/os_win/os_thread.c src/os_win/os_time.c src/os_win/os_vsnprintf.c diff --git a/src/third_party/wiredtiger/dist/api_config.py b/src/third_party/wiredtiger/dist/api_config.py index 8012a8a6be7..83a57ad4e0d 100644 --- a/src/third_party/wiredtiger/dist/api_config.py +++ b/src/third_party/wiredtiger/dist/api_config.py @@ -188,6 +188,8 @@ def get_default(c): t = gettype(c) if c.default == 'false': return '0' + elif t == 'string' and c.default == 'none': + return '' elif t == 'category': return '(%s)' % (','.join('%s=%s' % (subc.name, get_default(subc)) for subc in sorted(c.subconfig))) diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 8737c497140..7d9711c2b71 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -124,7 +124,7 @@ file_config = format_meta + [ items, and the default value of 4KB is a good choice absent requirements from the operating system or storage device''', min='512B', max='128MB'), - Config('block_compressor', '', r''' + Config('block_compressor', 'none', r''' configure a compressor for file blocks. Permitted values are \c "none" or custom compression engine name created with WT_CONNECTION::add_compressor. If WiredTiger has builtin support @@ -142,7 +142,7 @@ file_config = format_meta + [ applications which can rely on decompression to fail if a block has been corrupted''', choices=['on', 'off', 'uncompressed']), - Config('collator', '', r''' + Config('collator', 'none', r''' configure custom collation for keys. Permitted values are \c "none" or a custom collator name created with WT_CONNECTION::add_collator'''), @@ -154,11 +154,11 @@ file_config = format_meta + [ Config('format', 'btree', r''' the file format''', choices=['btree']), - Config('huffman_key', '', r''' + Config('huffman_key', 'none', r''' configure Huffman encoding for keys. Permitted values are \c "none", \c "english", \c "utf8<file>" or \c "utf16<file>". See @ref huffman for more information'''), - Config('huffman_value', '', r''' + Config('huffman_value', 'none', r''' configure Huffman encoding for values. Permitted values are \c "none", \c "english", \c "utf8<file>" or \c "utf16<file>". See @ref huffman for more information'''), @@ -273,7 +273,7 @@ table_only_config = [ ] index_only_config = [ - Config('extractor', '', r''' + Config('extractor', 'none', r''' configure custom extractor for indices. Permitted values are \c "none" or an extractor name created with WT_CONNECTION::add_extractor'''), @@ -426,7 +426,8 @@ connection_runtime_config = [ a timestamp prepended to each log record, may contain strftime conversion specifications'''), Config('wait', '0', r''' - seconds to wait between each write of the log records''', + seconds to wait between each write of the log records; setting + this value above 0 configures statistics logging''', min='0', max='100000'), ]), Config('verbose', '', r''' @@ -509,7 +510,7 @@ common_wiredtiger_open = [ Config('archive', 'true', r''' automatically archive unneeded log files''', type='boolean'), - Config('compressor', '', r''' + Config('compressor', 'none', r''' configure a compressor for log records. Permitted values are \c "none" or \c "bzip2", \c "snappy" or custom compression engine \c "name" created with WT_CONNECTION::add_compressor. diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 8929e0076ba..c2346caf1ea 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -1,5 +1,5 @@ # filelist -- -# List of source files for WiredTiger POSIX library. +# List of source files for WiredTiger library. src/async/async_api.c src/async/async_op.c @@ -109,6 +109,7 @@ src/os_posix/os_filesize.c src/os_posix/os_flock.c src/os_posix/os_fsync.c src/os_posix/os_ftruncate.c +src/os_posix/os_getenv.c src/os_posix/os_getline.c src/os_posix/os_getopt.c src/os_posix/os_map.c diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index aa5d65bcc8b..94f3ea6b6a5 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -138,6 +138,7 @@ GETTIMEOFDAY GIDs Gcc Geoff +GetEnvironmentVariableA GetFileAttributesEx GetFileSizeEx GetLastError @@ -639,6 +640,7 @@ func funcs gcc gdb +getenv getfiles getid getline diff --git a/src/third_party/wiredtiger/dist/s_win b/src/third_party/wiredtiger/dist/s_win index cb889f2ba49..187de91e498 100755 --- a/src/third_party/wiredtiger/dist/s_win +++ b/src/third_party/wiredtiger/dist/s_win @@ -3,19 +3,68 @@ t=__wt.$$ trap 'rm -f $t' 0 1 2 3 13 15 -egrep '#define|#undef' \ - ../build_posix/config.hin \ - ../build_win/wiredtiger_config.h | - sed 's/^.*#//' | - awk '{print $2}' | - egrep -v '^(LT_OBJDIR|PACKAGE|VERSION)' | - sort | uniq -u > $t - -test -s $t && { - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo 'Windows #defines missing from build_win/wiredtiger_config.h' - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - cat $t - exit 1 +# Insulate against locale-specific sort order +LC_ALL=C +export LC_ALL + +win_config() +{ + f='../build_win/wiredtiger_config.h' + egrep '#define|#undef' \ + ../build_posix/config.hin $f | + sed 's/^.*#//' | + awk '{print $2}' | + egrep -v '^(LT_OBJDIR|PACKAGE|VERSION)' | + sort | uniq -u > $t + + test -s $t && { + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Windows #defines missing from $f" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + cat $t + exit 1 + } } + +win_filelist() +{ + f='../build_win/filelist.win' + + # Process the files for which there's a Windows-specific version, then + # append Windows-only files. (There aren't yet any POSIX-only files.) + (sed \ + -e 's;os_posix/os_dir.c;os_win/os_dir.c;' \ + -e 's;os_posix/os_dlopen.c;os_win/os_dlopen.c;' \ + -e 's;os_posix/os_dlopen.c;os_win/os_dlopen.c;' \ + -e 's;os_posix/os_dlopen.c;os_win/os_dlopen.c;' \ + -e 's;os_posix/os_errno.c;os_win/os_errno.c;' \ + -e 's;os_posix/os_exist.c;os_win/os_exist.c;' \ + -e 's;os_posix/os_fallocate.c;os_win/os_fallocate.c;' \ + -e 's;os_posix/os_filesize.c;os_win/os_filesize.c;' \ + -e 's;os_posix/os_flock.c;os_win/os_flock.c;' \ + -e 's;os_posix/os_fsync.c;os_win/os_fsync.c;' \ + -e 's;os_posix/os_ftruncate.c;os_win/os_ftruncate.c;' \ + -e 's;os_posix/os_getenv.c;os_win/os_getenv.c;' \ + -e 's;os_posix/os_map.c;os_win/os_map.c;' \ + -e 's;os_posix/os_mtx_cond.c;os_win/os_mtx_cond.c;' \ + -e 's;os_posix/os_once.c;os_win/os_once.c;' \ + -e 's;os_posix/os_open.c;os_win/os_open.c;' \ + -e 's;os_posix/os_path.c;os_win/os_path.c;' \ + -e 's;os_posix/os_priv.c;os_win/os_priv.c;' \ + -e 's;os_posix/os_remove.c;os_win/os_remove.c;' \ + -e 's;os_posix/os_rename.c;os_win/os_rename.c;' \ + -e 's;os_posix/os_rw.c;os_win/os_rw.c;' \ + -e 's;os_posix/os_sleep.c;os_win/os_sleep.c;' \ + -e 's;os_posix/os_thread.c;os_win/os_thread.c;' \ + -e 's;os_posix/os_time.c;os_win/os_time.c;' \ + -e 's;os_posix/os_yield.c;os_win/os_yield.c;' + echo 'src/os_win/os_snprintf.c' + echo 'src/os_win/os_vsnprintf.c') < filelist | sort > $t + cmp $t $f > /dev/null 2>&1 || + (echo "Building $f" && rm -f $f && cp $t $f) +} + +win_config +win_filelist + exit 0 diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/packing-test.py b/src/third_party/wiredtiger/lang/python/wiredtiger/packing-test.py index eaae4d51ae7..4b06b2bc608 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/packing-test.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/packing-test.py @@ -28,11 +28,50 @@ from packing import pack, unpack +def check_common(fmt, verbose, *v): + v = list(v) + packed = pack(fmt, *v) + unpacked = unpack(fmt, packed) + if unpacked == v: + result = 'ok' + else: + result = '** FAIL!' + print '* %s as %s: %s' % (repr(v), fmt, result) + if verbose or unpacked != v: + print '** packed: ', ''.join('%02x' % ord(c) for c in packed) + print '** unpacked: ', unpacked + def check(fmt, *v): - print fmt, repr(v), ''.join('%02x' % ord(c) for c in pack(fmt, *v)) + check_common(fmt, False, *v) + +def check_verbose(fmt, *v): + check_common(fmt, True, *v) + + +if __name__ == '__main__': + import sys + if 'verbose' in sys.argv: + check = check_verbose + check('iii', 0, 101, -99) + check('3i', 0, 101, -99) + check('iS', 42, "forty two") + + # + check('S', 'abc') + check('9S', 'a' * 9) + check('9SS', "forty two", "spam egg") + check('42S', 'a' * 42) + check('42SS', 'a' * 42, 'something') + check('S42S', 'something', 'a' * 42) + # nul terminated string with padding + check('10SS', 'aaaaa\x00\x00\x00\x00\x00', 'something') + check('S10S', 'something', 'aaaaa\x00\x00\x00\x00\x00') + + check('u', r"\x42" * 20) + check('uu', r"\x42" * 10, r"\x42" * 10) + check('3u', r"\x4") + check('3uu', r"\x4", r"\x42" * 10) + check('u3u', r"\x42" * 10, r"\x4") -check('iii', 0, 101, -99) -check('3i', 0, 101, -99) -check('iS', 42, "forty two") -check('u', r"\x42" * 20) -check('uu', r"\x42" * 10, r"\x42" * 10) + check('s', "4") + check("2s", "42") diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py index 6bcb82e76d5..ee12434cabd 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py @@ -27,6 +27,27 @@ # # WiredTiger variable-length packing and unpacking functions +"""Packing and unpacking functions +The format string uses the following conversions: +Format Python Notes + x N/A pad byte, no associated value + b int signed byte + B int unsigned byte + h int signed 16-bit + H int unsigned 16-bit + i int signed 32-bit + I int unsigned 32-bit + l int signed 32-bit + L int unsigned 32-bit + q int signed 64-bit + Q int unsigned 64-bit + r int record number + s str fixed-length string + S str NUL-terminated string + t int fixed-length bit field + u str raw byte array +""" + from intpacking import pack_int, unpack_int def __get_type(fmt): @@ -40,6 +61,18 @@ def __get_type(fmt): tfmt = '.' return tfmt, fmt +def __unpack_iter_fmt(fmt): + size = 0 + havesize = 0 + for offset, char in enumerate(fmt): + if char.isdigit(): + size = (size * 10) + int(char) + havesize = 1 + else: + yield offset, havesize, size, char + size = 0 + havesize = 0 + def unpack(fmt, s): tfmt, fmt = __get_type(fmt) if not fmt: @@ -47,13 +80,8 @@ def unpack(fmt, s): if tfmt != '.': raise ValueError('Only variable-length encoding is currently supported') result = [] - havesize = size = 0 - for offset, f in enumerate(fmt): - if f.isdigit(): - size = (size * 10) + int(f) - havesize = 1 - continue - elif f == 'x': + for offset, havesize, size, f in __unpack_iter_fmt(fmt): + if f == 'x': if not havesize: size = 1 s = s[size:] @@ -86,9 +114,23 @@ def unpack(fmt, s): for j in xrange(size): v, s = unpack_int(s) result.append(v) - havesize = size = 0 return result +def __pack_iter_fmt(fmt, values): + index = 0 + for offset, havesize, size, char in __unpack_iter_fmt(fmt): + if char == 'x': # padding no value + yield offset, havesize, size, char, None + elif char in 'Ssut': + yield offset, havesize, size, char, values[index] + index += 1 + else: # integral type + size = size if havesize else 1 + for i in xrange(size): + value = values[index] + yield offset, havesize, 1, char, value + index = index + 1 + def pack(fmt, *values): tfmt, fmt = __get_type(fmt) if not fmt: @@ -96,20 +138,15 @@ def pack(fmt, *values): if tfmt != '.': raise ValueError('Only variable-length encoding is currently supported') result = '' - havesize = i = size = 0 - for offset, f in enumerate(fmt): - if f.isdigit(): - size = (size * 10) + int(f) - havesize = 1 - continue - elif f == 'x': + i = 0 + for offset, havesize, size, f, val in __pack_iter_fmt(fmt, values): + if f == 'x': if not havesize: result += '\0' else: result += '\0' * size # Note: no value, don't increment i elif f in 'Ssu': - val = values[i] if f == 'S' and '\0' in val: l = val.find('\0') else: @@ -129,7 +166,6 @@ def pack(fmt, *values): result += '\0' elif size > l: result += '\0' * (size - l) - i += 1 elif f in 't': # bit type, size is number of bits if not havesize: @@ -137,17 +173,10 @@ def pack(fmt, *values): if size > 8: raise ValueError("bit count cannot be greater than 8 for 't' encoding") mask = (1 << size) - 1 - val = values[i] if (mask & val) != val: raise ValueError("value out of range for 't' encoding") result += chr(val) - i += 1 else: # integral type - if not havesize: - size = 1 - for j in xrange(size): - result += pack_int(values[i]) - i += 1 - havesize = size = 0 + result += pack_int(val) return result diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index c528ee4a6aa..c8a26e77341 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -168,6 +168,9 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, "off %" PRIuMAX ", size %" PRIu32 ", cksum %" PRIu32, (uintmax_t)offset, size, cksum)); + WT_STAT_FAST_CONN_INCR(session, block_read); + WT_STAT_FAST_CONN_INCRV(session, block_byte_read, size); + /* * Grow the buffer as necessary and read the block. Buffers should be * aligned for reading, but there are lots of buffers (for example, file @@ -188,25 +191,24 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, buf->size = size; blk = WT_BLOCK_HEADER_REF(buf->mem); - blk->cksum = 0; - page_cksum = __wt_cksum(buf->mem, - F_ISSET(blk, WT_BLOCK_DATA_CKSUM) ? size : WT_BLOCK_COMPRESS_SKIP); - if (cksum != page_cksum) { - if (!F_ISSET(session, WT_SESSION_SALVAGE_CORRUPT_OK)) - __wt_errx(session, - "read checksum error [%" - PRIu32 "B @ %" PRIuMAX ", %" - PRIu32 " != %" PRIu32 "]", - size, (uintmax_t)offset, cksum, page_cksum); - - /* Panic if a checksum fails during an ordinary read. */ - return (block->verify || - F_ISSET(session, WT_SESSION_SALVAGE_CORRUPT_OK) ? - WT_ERROR : - __wt_illegal_value(session, block->name)); + page_cksum = blk->cksum; + if (page_cksum == cksum) { + blk->cksum = 0; + page_cksum = __wt_cksum(buf->mem, + F_ISSET(blk, WT_BLOCK_DATA_CKSUM) ? + size : WT_BLOCK_COMPRESS_SKIP); + if (page_cksum == cksum) + return (0); } - WT_STAT_FAST_CONN_INCR(session, block_read); - WT_STAT_FAST_CONN_INCRV(session, block_byte_read, size); - return (0); + if (!F_ISSET(session, WT_SESSION_SALVAGE_CORRUPT_OK)) + __wt_errx(session, + "read checksum error [%" PRIu32 "B @ %" PRIuMAX ", %" + PRIu32 " != %" PRIu32 "]", + size, (uintmax_t)offset, cksum, page_cksum); + + /* Panic if a checksum fails during an ordinary read. */ + return (block->verify || + F_ISSET(session, WT_SESSION_SALVAGE_CORRUPT_OK) ? + WT_ERROR : __wt_illegal_value(session, block->name)); } diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index ebbb335d3a8..05407537a53 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -453,7 +453,8 @@ __wt_debug_tree_shape( if (page == NULL) page = S2BT(session)->root.page; - __debug_tree_shape_worker(ds, page, 0); + WT_WITH_PAGE_INDEX(session, + __debug_tree_shape_worker(ds, page, 0)); __dmsg_wrapup(ds); return (0); @@ -535,6 +536,7 @@ __debug_tree( static int __debug_page(WT_DBG *ds, WT_PAGE *page, uint32_t flags) { + WT_DECL_RET; WT_SESSION_IMPL *session; session = ds->session; @@ -549,14 +551,18 @@ __debug_page(WT_DBG *ds, WT_PAGE *page, uint32_t flags) __debug_page_col_fix(ds, page); break; case WT_PAGE_COL_INT: - WT_RET(__debug_page_col_int(ds, page, flags)); + WT_WITH_PAGE_INDEX(session, + ret = __debug_page_col_int(ds, page, flags)); + WT_RET(ret); break; case WT_PAGE_COL_VAR: if (LF_ISSET(WT_DEBUG_TREE_LEAF)) WT_RET(__debug_page_col_var(ds, page)); break; case WT_PAGE_ROW_INT: - WT_RET(__debug_page_row_int(ds, page, flags)); + WT_WITH_PAGE_INDEX(session, + ret = __debug_page_row_int(ds, page, flags)); + WT_RET(ret); break; case WT_PAGE_ROW_LEAF: if (LF_ISSET(WT_DEBUG_TREE_LEAF)) diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 6d69bd8fc74..69c9ab27656 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -106,7 +106,9 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) session, root_addr, root_addr_size)); /* Warm the cache, if possible. */ - WT_ERR(__btree_preload(session)); + WT_WITH_PAGE_INDEX(session, + ret = __btree_preload(session)); + WT_ERR(ret); /* Get the last record number in a column-store file. */ if (btree->type != BTREE_ROW) diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index 8ab5a6a30c0..9d154311a8c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -295,8 +295,8 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_STAT_FAST_CONN_INCR(session, cache_write); WT_STAT_FAST_DATA_INCR(session, cache_write); - WT_STAT_FAST_CONN_INCRV(session, cache_bytes_write, ip->size); - WT_STAT_FAST_DATA_INCRV(session, cache_bytes_write, ip->size); + WT_STAT_FAST_CONN_INCRV(session, cache_bytes_write, dsk->mem_size); + WT_STAT_FAST_DATA_INCRV(session, cache_bytes_write, dsk->mem_size); err: __wt_scr_free(&tmp); return (ret); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index c6b97733b69..a14e82d980d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -716,11 +716,10 @@ __split_multi_inmem( /* * We modified the page above, which will have set the first dirty * transaction to the last transaction current running. However, the - * updates we installed may be older than that. Take the oldest active - * transaction ID to make sure these updates are not skipped by a - * checkpoint. + * updates we installed may be older than that. Inherit the first + * dirty transaction from the original page. */ - page->modify->first_dirty_txn = S2C(session)->txn_global.oldest_id; + page->modify->first_dirty_txn = orig->modify->first_dirty_txn; err: /* Free any resources that may have been cached in the cursor. */ WT_TRET(__wt_btcur_close(&cbt)); @@ -900,7 +899,8 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, alloc_index = NULL; #ifdef HAVE_DIAGNOSTIC - __split_verify_intl_key_order(session, parent); + WT_WITH_PAGE_INDEX(session, + __split_verify_intl_key_order(session, parent)); #endif /* @@ -977,7 +977,8 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, * are holding it locked. */ if (ret == 0 && !exclusive && __split_should_deepen(session, parent)) - ret = __split_deepen(session, parent); + WT_WITH_PAGE_INDEX(session, + ret = __split_deepen(session, parent)); err: if (locked) F_CLR_ATOMIC(parent, WT_PAGE_SPLITTING); @@ -1135,8 +1136,8 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref, int *splitp) /* * We modified the page above, which will have set the first dirty * transaction to the last transaction current running. However, the - * updates we are moving may be older than that: inherit the original - * page's transaction ID. + * updates we installed may be older than that. Inherit the first + * dirty transaction from the original page. */ right->modify->first_dirty_txn = page->modify->first_dirty_txn; diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index e7caf02fd2f..869fa2bb696 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -122,7 +122,8 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) __wt_page_type_string( btree->root.page->type))); #endif - ret = __verify_tree(session, &btree->root, vs); + WT_WITH_PAGE_INDEX(session, + ret = __verify_tree(session, &btree->root, vs)); WT_TRET(__wt_cache_op(session, NULL, WT_SYNC_DISCARD)); } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 6bd4edd3543..b45cdf0b75a 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -547,7 +547,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "error_prefix=,eviction=(threads_max=1,threads_min=1)," "eviction_dirty_target=80,eviction_target=80,eviction_trigger=95," "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=," - "shared_cache=(chunk=10MB,name=none,reserve=0,size=500MB)," + "shared_cache=(chunk=10MB,name=,reserve=0,size=500MB)," "statistics=none,statistics_log=(on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", @@ -666,7 +666,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_extend=,hazard_max=1000,log=(archive=,compressor=,enabled=0" ",file_max=100MB,path=,prealloc=),lsm_manager=(merge=," "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," - "session_max=100,shared_cache=(chunk=10MB,name=none,reserve=0," + "session_max=100,shared_cache=(chunk=10MB,name=,reserve=0," "size=500MB),statistics=none,statistics_log=(on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" @@ -683,7 +683,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_extend=,hazard_max=1000,log=(archive=,compressor=,enabled=0" ",file_max=100MB,path=,prealloc=),lsm_manager=(merge=," "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," - "session_max=100,shared_cache=(chunk=10MB,name=none,reserve=0," + "session_max=100,shared_cache=(chunk=10MB,name=,reserve=0," "size=500MB),statistics=none,statistics_log=(on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" @@ -700,12 +700,11 @@ static const WT_CONFIG_ENTRY config_entries[] = { "extensions=,file_extend=,hazard_max=1000,log=(archive=," "compressor=,enabled=0,file_max=100MB,path=,prealloc=)," "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," - "multiprocess=0,session_max=100,shared_cache=(chunk=10MB," - "name=none,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),verbose=," - "version=(major=0,minor=0)", + "multiprocess=0,session_max=100,shared_cache=(chunk=10MB,name=," + "reserve=0,size=500MB),statistics=none,statistics_log=(on_close=0" + ",path=\"WiredTigerStat.%d.%H\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),verbose=,version=(major=0,minor=0)", confchk_wiredtiger_open_basecfg }, { "wiredtiger_open_usercfg", @@ -717,11 +716,11 @@ static const WT_CONFIG_ENTRY config_entries[] = { "extensions=,file_extend=,hazard_max=1000,log=(archive=," "compressor=,enabled=0,file_max=100MB,path=,prealloc=)," "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,mmap=," - "multiprocess=0,session_max=100,shared_cache=(chunk=10MB," - "name=none,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),verbose=", + "multiprocess=0,session_max=100,shared_cache=(chunk=10MB,name=," + "reserve=0,size=500MB),statistics=none,statistics_log=(on_close=0" + ",path=\"WiredTigerStat.%d.%H\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),verbose=", confchk_wiredtiger_open_usercfg }, { NULL, NULL, NULL } diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index cc88f848861..c878024edb4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -519,12 +519,11 @@ err: if (nextractor != NULL) { * Given a configuration, configure the extractor. */ int -__wt_extractor_config(WT_SESSION_IMPL *session, const char *config, - WT_EXTRACTOR **extractorp, int *ownp) +__wt_extractor_config(WT_SESSION_IMPL *session, + const char *config, WT_EXTRACTOR **extractorp, int *ownp) { WT_CONNECTION_IMPL *conn; WT_CONFIG_ITEM cval; - WT_DECL_RET; WT_NAMED_EXTRACTOR *nextractor; *extractorp = NULL; @@ -532,35 +531,32 @@ __wt_extractor_config(WT_SESSION_IMPL *session, const char *config, conn = S2C(session); - if ((ret = - __wt_config_getones_none(session, config, "extractor", &cval)) != 0) - return (ret == WT_NOTFOUND || cval.len == 0 ? 0 : ret); - - if (cval.len > 0) { - TAILQ_FOREACH(nextractor, &conn->extractorqh, q) - if (WT_STRING_MATCH( - nextractor->name, cval.str, cval.len)) - break; + WT_RET_NOTFOUND_OK( + __wt_config_getones_none(session, config, "extractor", &cval)); + if (cval.len == 0) + return (0); - if (nextractor == NULL) - WT_RET_MSG(session, EINVAL, - "unknown extractor '%.*s'", - (int)cval.len, cval.str); - - if (nextractor->extractor->customize != NULL) { - WT_RET(__wt_config_getones(session, - config, "app_metadata", &cval)); - WT_RET(nextractor->extractor->customize( - nextractor->extractor, &session->iface, - session->dhandle->name, &cval, extractorp)); - } + TAILQ_FOREACH(nextractor, &conn->extractorqh, q) + if (WT_STRING_MATCH(nextractor->name, cval.str, cval.len)) + break; - if (*extractorp == NULL) - *extractorp = nextractor->extractor; - else - *ownp = 1; + if (nextractor == NULL) + WT_RET_MSG(session, EINVAL, + "unknown extractor '%.*s'", (int)cval.len, cval.str); + + if (nextractor->extractor->customize != NULL) { + WT_RET(__wt_config_getones(session, + config, "app_metadata", &cval)); + WT_RET(nextractor->extractor->customize( + nextractor->extractor, &session->iface, + session->dhandle->name, &cval, extractorp)); } + if (*extractorp == NULL) + *extractorp = nextractor->extractor; + else + *ownp = 1; + return (0); } @@ -1007,15 +1003,19 @@ static int __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf) { WT_CONFIG_ITEM cval; + WT_DECL_RET; const char *env_config; size_t len; - if ((env_config = getenv("WIREDTIGER_CONFIG")) == NULL) + ret = __wt_getenv(session, "WIREDTIGER_CONFIG", &env_config); + if (ret == WT_NOTFOUND) return (0); + WT_ERR(ret); + len = strlen(env_config); if (len == 0) - return (0); - WT_RET(__wt_buf_set(session, cbuf, env_config, len + 1)); + goto err; /* Free the memory. */ + WT_ERR(__wt_buf_set(session, cbuf, env_config, len + 1)); /* * Security stuff: @@ -1023,26 +1023,28 @@ __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf) * If the "use_environment_priv" configuration string is set, use the * environment variable if the process has appropriate privileges. */ - WT_RET(__wt_config_gets(session, cfg, "use_environment_priv", &cval)); + WT_ERR(__wt_config_gets(session, cfg, "use_environment_priv", &cval)); if (cval.val == 0 && __wt_has_priv()) - WT_RET_MSG(session, WT_ERROR, "%s", + WT_ERR_MSG(session, WT_ERROR, "%s", "WIREDTIGER_CONFIG environment variable set but process " "lacks privileges to use that environment variable"); /* Check any version. */ - WT_RET(__conn_config_check_version(session, env_config)); + WT_ERR(__conn_config_check_version(session, env_config)); /* Upgrade the configuration string. */ - WT_RET(__wt_config_upgrade(session, cbuf)); + WT_ERR(__wt_config_upgrade(session, cbuf)); /* Check the configuration information. */ - WT_RET(__wt_config_check(session, + WT_ERR(__wt_config_check(session, WT_CONFIG_REF(session, wiredtiger_open), env_config, 0)); /* Append it to the stack. */ - __conn_config_append(cfg, env_config); + __conn_config_append(cfg, cbuf->data); - return (0); +err: __wt_free(session, env_config); + + return (ret); } /* @@ -1052,17 +1054,19 @@ __conn_config_env(WT_SESSION_IMPL *session, const char *cfg[], WT_ITEM *cbuf) static int __conn_home(WT_SESSION_IMPL *session, const char *home, const char *cfg[]) { + WT_DECL_RET; WT_CONFIG_ITEM cval; /* If the application specifies a home directory, use it. */ if (home != NULL) goto copy; + ret = __wt_getenv(session, "WIREDTIGER_HOME", &S2C(session)->home); + if (ret == 0) + return (0); + /* If there's no WIREDTIGER_HOME environment variable, use ".". */ - if ((home = getenv("WIREDTIGER_HOME")) == NULL || strlen(home) == 0) { - home = "."; - goto copy; - } + home = "."; /* * Security stuff: diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 5d5a67ccac8..184fac26d7c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -667,7 +667,8 @@ __conn_dhandle_remove(WT_SESSION_IMPL *session, int final) WT_ASSERT(session, F_ISSET(session, WT_SESSION_HANDLE_LIST_LOCKED)); /* Check if the handle was reacquired by a session while we waited. */ - if (!final && dhandle->session_ref != 0) + if (!final && + (dhandle->session_inuse != 0 || dhandle->session_ref != 0)) return (EBUSY); WT_CONN_DHANDLE_REMOVE(conn, dhandle, bucket); @@ -715,7 +716,7 @@ __wt_conn_dhandle_discard_single(WT_SESSION_IMPL *session, int final) __wt_spin_destroy(session, &dhandle->close_lock); __wt_overwrite_and_free(session, dhandle); - WT_CLEAR_BTREE_IN_SESSION(session); + session->dhandle = NULL; } return (ret); diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index a1ed9ca11e5..5916043e44f 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -41,30 +41,17 @@ __sweep(WT_SESSION_IMPL *session) continue; /* - * We have a candidate for closing; if it's open, flush dirty - * leaf pages, then acquire an exclusive lock on the handle - * and close it. We might be blocking opens for a long time - * (over disk I/O), but the handle was quiescent for awhile. + * We have a candidate for closing; if it's open, acquire an + * exclusive lock on the handle and close it. We might be + * blocking opens for a long time (over disk I/O), but the + * handle was quiescent for awhile. + * + * The close can fail if an update cannot be written (updates + * in a no-longer-referenced file might not yet be globally + * visible if sessions have disjoint sets of files open). If + * the handle is busy, skip it, we'll retry the close the next + * time, after the transaction state has progressed. * - * The close can fail if an update cannot be written (updates in - * a no-longer-referenced file might not yet be globally visible - * if sessions have disjoint sets of files open). If the handle - * is busy, skip it, we'll retry the close the next time, after - * the transaction state has progressed. - */ - if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { - WT_WITH_DHANDLE(session, dhandle, - ret = __wt_cache_op( - session, NULL, WT_SYNC_WRITE_LEAVES)); - WT_RET(ret); - - /* Re-check that this looks like a good candidate. */ - if (dhandle->timeofdeath == 0 || - now <= dhandle->timeofdeath + WT_DHANDLE_SWEEP_WAIT) - continue; - } - - /* * We don't set WT_DHANDLE_EXCLUSIVE deliberately, we want * opens to block on us rather than returning an EBUSY error to * the application. @@ -79,9 +66,11 @@ __sweep(WT_SESSION_IMPL *session) if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { WT_WITH_DHANDLE(session, dhandle, ret = __wt_conn_btree_sync_and_close(session, 0)); - if (ret == 0) - WT_STAT_FAST_CONN_INCR( - session, dh_conn_handles); + if (ret != 0) + goto unlock; + + /* We closed the btree handle, bump the statistic. */ + WT_STAT_FAST_CONN_INCR(session, dh_conn_handles); } /* @@ -90,17 +79,18 @@ __sweep(WT_SESSION_IMPL *session) * re-checks that the handle is not in use, which is why we * don't do any special handling of EBUSY returns above. */ - if (ret == 0 && dhandle->session_ref == 0) { + if (dhandle->session_inuse == 0 && dhandle->session_ref == 0) { WT_WITH_DHANDLE(session, dhandle, ret = __wt_conn_dhandle_discard_single(session, 0)); + if (ret != 0) + goto unlock; /* If the handle was discarded, it isn't locked. */ - if (ret == 0) - locked = 0; + locked = 0; } else WT_STAT_FAST_CONN_INCR(session, dh_conn_ref); - if (locked) +unlock: if (locked) WT_TRET(__wt_writeunlock(session, dhandle->rwlock)); WT_RET_BUSY_OK(ret); diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 9037a4a1fcc..69fa1f85c93 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -407,6 +407,15 @@ __evict_has_work(WT_SESSION_IMPL *session, uint32_t *flagsp) (cache->eviction_dirty_target * bytes_max) / 100) /* Ignore clean pages unless the cache is too large */ LF_SET(WT_EVICT_PASS_DIRTY); + else if (F_ISSET(cache, WT_EVICT_WOULD_BLOCK)) { + /* + * Evict pages with oldest generation (which would otherwise + * block application threads) set regardless of whether we have + * reached the eviction trigger. + */ + LF_SET(WT_EVICT_PASS_WOULD_BLOCK); + F_CLR(cache, WT_EVICT_WOULD_BLOCK); + } if (F_ISSET(cache, WT_EVICT_STUCK)) LF_SET(WT_EVICT_PASS_AGGRESSIVE); @@ -941,9 +950,11 @@ retry: while (slot < max_entries && ret == 0) { * Re-check the "no eviction" flag -- it is used to enforce * exclusive access when a handle is being closed. */ - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) + if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { WT_WITH_BTREE(session, btree, ret = __evict_walk_file(session, &slot, flags)); + WT_ASSERT(session, session->split_gen == 0); + } __wt_spin_unlock(session, &cache->evict_walk_lock); @@ -1074,6 +1085,14 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp, uint32_t flags) if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)) continue; + /* + * If we are only trickling out pages marked for definite + * eviction, skip anything that isn't marked. + */ + if (LF_ISSET(WT_EVICT_PASS_WOULD_BLOCK) && + page->read_gen != WT_READGEN_OLDEST) + continue; + /* Limit internal pages to 50% unless we get aggressive. */ if ((page->type == WT_PAGE_COL_INT || page->type == WT_PAGE_ROW_INT) && @@ -1281,6 +1300,7 @@ __wt_evict_lru_page(WT_SESSION_IMPL *session, int is_app) page->read_gen = __wt_cache_read_gen_set(session); WT_WITH_BTREE(session, btree, ret = __wt_evict_page(session, ref)); + WT_ASSERT(session, is_app || session->split_gen == 0); (void)WT_ATOMIC_SUB4(btree->evict_busy, 1); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index bc791de6d0f..4e96898fd92 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -24,9 +24,10 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_TXN_STATE *txn_state; - int inmem_split, istree; + int forced_eviction, inmem_split, istree; page = ref->page; + forced_eviction = (page->read_gen == WT_READGEN_OLDEST); inmem_split = istree = 0; WT_RET(__wt_verbose(session, WT_VERB_EVICT, @@ -75,7 +76,8 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive) /* Discard any subtree rooted in this page. */ if (istree) - __evict_discard_tree(session, ref, exclusive, 1); + WT_WITH_PAGE_INDEX(session, + __evict_discard_tree(session, ref, exclusive, 1)); /* Update the reference and discard the page. */ if (mod == NULL || !F_ISSET(mod, WT_PM_REC_MASK)) { @@ -115,6 +117,12 @@ done: session->excl_next = 0; if (txn_state != NULL) txn_state->snap_min = WT_TXN_NONE; + if ((inmem_split || (forced_eviction && ret == EBUSY)) && + !F_ISSET(S2C(session)->cache, WT_EVICT_WOULD_BLOCK)) { + F_SET(S2C(session)->cache, WT_EVICT_WOULD_BLOCK); + WT_TRET(__wt_evict_server_wake(session)); + } + return (ret); } @@ -253,6 +261,45 @@ __evict_discard_tree( } /* + * __evict_review_subtree -- + * Review a subtree for conditions that would block its eviction. + */ +static int +__evict_review_subtree(WT_SESSION_IMPL *session, + WT_REF *ref, int exclusive, int *inmem_splitp, int *istreep) +{ + WT_PAGE *page; + WT_REF *child; + + page = ref->page; + + WT_INTL_FOREACH_BEGIN(session, page, child) { + switch (child->state) { + case WT_REF_DISK: /* On-disk */ + case WT_REF_DELETED: /* On-disk, deleted */ + break; + case WT_REF_MEM: /* In-memory */ + /* + * Tell our caller if there's a subtree so we + * know to do a full walk when discarding the + * page. + */ + *istreep = 1; + WT_RET(__evict_review(session, child, exclusive, + 0, inmem_splitp, istreep)); + break; + case WT_REF_LOCKED: /* Being evicted */ + case WT_REF_READING: /* Being read */ + case WT_REF_SPLIT: /* Being split */ + return (EBUSY); + WT_ILLEGAL_VALUE(session); + } + } WT_INTL_FOREACH_END; + + return (0); +} + +/* * __evict_review -- * Get exclusive access to the page and review the page and its subtree * for conditions that would block its eviction. @@ -262,13 +309,12 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, int exclusive, int top, int *inmem_splitp, int *istreep) { WT_BTREE *btree; + WT_DECL_RET; WT_PAGE *page; WT_PAGE_MODIFY *mod; - WT_REF *child; uint32_t flags; btree = S2BT(session); - page = ref->page; /* * Get exclusive access to the page if our caller doesn't have the tree @@ -287,36 +333,20 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, __wt_evict_list_clear_page(session, ref); } + /* Now that we have exclusive access, review the page. */ + page = ref->page; + mod = page->modify; + /* * Recurse through the page's subtree: this happens first because we * have to write pages in depth-first order, otherwise we'll dirty * pages after we've written them. */ - if (WT_PAGE_IS_INTERNAL(page)) - WT_INTL_FOREACH_BEGIN(session, page, child) { - switch (child->state) { - case WT_REF_DISK: /* On-disk */ - case WT_REF_DELETED: /* On-disk, deleted */ - break; - case WT_REF_MEM: /* In-memory */ - /* - * Tell our caller if there's a subtree so we - * know to do a full walk when discarding the - * page. - */ - *istreep = 1; - WT_RET(__evict_review(session, child, exclusive, - 0, inmem_splitp, istreep)); - break; - case WT_REF_LOCKED: /* Being evicted */ - case WT_REF_READING: /* Being read */ - case WT_REF_SPLIT: /* Being split */ - return (EBUSY); - WT_ILLEGAL_VALUE(session); - } - } WT_INTL_FOREACH_END; - - mod = page->modify; + if (WT_PAGE_IS_INTERNAL(page)) { + WT_WITH_PAGE_INDEX(session, ret = __evict_review_subtree( + session, ref, exclusive, inmem_splitp, istreep)); + WT_RET(ret); + } /* * If the tree was deepened, there's a requirement that newly created @@ -427,7 +457,7 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, if (exclusive) LF_SET(WT_SKIP_UPDATE_ERR); else if (top && !WT_PAGE_IS_INTERNAL(page) && - page->memory_footprint > 10 * btree->maxleafpage) + page->read_gen == WT_READGEN_OLDEST) LF_SET(WT_SKIP_UPDATE_RESTORE); WT_RET(__wt_reconcile(session, ref, NULL, flags)); WT_ASSERT(session, diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 8a3ec9ade89..72feb3dce8f 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -424,17 +424,15 @@ struct __wt_page { #define WT_INTL_FOREACH_BEGIN(session, page, ref) do { \ WT_PAGE_INDEX *__pindex; \ WT_REF **__refp; \ - WT_SESSION_IMPL *__session = (session); \ uint32_t __entries; \ - WT_ENTER_PAGE_INDEX(session); \ + WT_ASSERT(session, session->split_gen != 0); \ for (__pindex = WT_INTL_INDEX_COPY(page), \ __refp = __pindex->index, \ __entries = __pindex->entries; __entries > 0; --__entries) {\ (ref) = *__refp++; #define WT_INTL_FOREACH_END \ - } \ - WT_LEAVE_PAGE_INDEX(__session); \ - } while (0) + } \ +} while (0) /* Row-store leaf page. */ struct { diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index a6425e97837..cf688b3993f 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -19,6 +19,7 @@ #define WT_EVICT_PASS_AGGRESSIVE 0x01 #define WT_EVICT_PASS_ALL 0x02 #define WT_EVICT_PASS_DIRTY 0x04 +#define WT_EVICT_PASS_WOULD_BLOCK 0x08 /* * WT_EVICT_ENTRY -- @@ -33,7 +34,6 @@ struct __wt_evict_entry { * WT_EVICT_WORKER -- * Encapsulation of an eviction worker thread. */ - struct __wt_evict_worker { WT_SESSION_IMPL *session; u_int id; @@ -114,6 +114,7 @@ struct __wt_cache { #define WT_EVICT_ACTIVE 0x04 /* Eviction server is active */ #define WT_EVICT_CLEAR_WALKS 0x08 /* Clear eviction walks */ #define WT_EVICT_STUCK 0x10 /* Eviction server is stuck */ +#define WT_EVICT_WOULD_BLOCK 0x20 /* Pages that would block apps */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 8cd7c9ba90b..1a6839b11fe 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -5,19 +5,6 @@ * See the file LICENSE for redistribution information. */ -/* - * XXX - * The server threads use their own WT_SESSION_IMPL handles because they may - * want to block (for example, the eviction server calls reconciliation, and - * some of the reconciliation diagnostic code reads pages), and the user's - * session handle is already blocking on a server thread. The problem is the - * server thread needs to reference the correct btree handle, and that's - * hanging off the application's thread of control. For now, I'm just making - * it obvious where that's getting done. - */ -#define WT_SET_BTREE_IN_SESSION(s, b) ((s)->dhandle = b->dhandle) -#define WT_CLEAR_BTREE_IN_SESSION(s) ((s)->dhandle = NULL) - #define WT_WITH_DHANDLE(s, d, e) do { \ WT_DATA_HANDLE *__saved_dhandle = (s)->dhandle; \ (s)->dhandle = (d); \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index bb7caa991d6..056058f3828 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -442,6 +442,7 @@ extern int __wt_directory_sync(WT_SESSION_IMPL *session, char *path); extern int __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh); extern int __wt_fsync_async(WT_SESSION_IMPL *session, WT_FH *fh); extern int __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t len); +extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp); extern int __wt_getline(WT_SESSION_IMPL *session, WT_ITEM *buf, FILE *fp); extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr); extern int __wt_mmap(WT_SESSION_IMPL *session, WT_FH *fh, void *mapp, size_t *lenp, void **mappingcookie); diff --git a/src/third_party/wiredtiger/src/include/os_windows.h b/src/third_party/wiredtiger/src/include/os_windows.h index b4ff458a10f..aa39f1f44ab 100644 --- a/src/third_party/wiredtiger/src/include/os_windows.h +++ b/src/third_party/wiredtiger/src/include/os_windows.h @@ -30,9 +30,14 @@ typedef uint32_t u_int; typedef unsigned char u_char; typedef unsigned long u_long; -/* < VS 2013 is not C99 compat */ +/* <= VS 2013 is not C99 compat */ #if _MSC_VER < 1900 -#define snprintf _snprintf +#define snprintf _wt_snprintf + +_Check_return_opt_ int __cdecl _wt_snprintf( + _Out_writes_(_MaxCount) char * _DstBuf, + _In_ size_t _MaxCount, + _In_z_ _Printf_format_string_ const char * _Format, ...); #endif /* diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index df2872dcb4d..075514e02b4 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -946,7 +946,7 @@ struct __wt_session { * created with WT_CONNECTION::add_compressor. If WiredTiger has * builtin support for \c "snappy" or \c "zlib" compression\, these * names are also available. See @ref compression for more - * information., a string; default empty.} + * information., a string; default \c none.} * @config{cache_resident, do not ever evict the object's pages; see * @ref tuning_cache_resident for more information., a boolean flag; * default \c false.} @@ -966,7 +966,7 @@ struct __wt_session { * strings; default empty.} * @config{collator, configure custom collation for keys. Permitted * values are \c "none" or a custom collator name created with - * WT_CONNECTION::add_collator., a string; default empty.} + * WT_CONNECTION::add_collator., a string; default \c none.} * @config{columns, list of the column names. Comma-separated list of * the form <code>(column[\,...])</code>. For tables\, the number of * entries must match the total number of values in \c key_format and \c @@ -982,17 +982,17 @@ struct __wt_session { * specified configuration., a boolean flag; default \c false.} * @config{extractor, configure custom extractor for indices. Permitted * values are \c "none" or an extractor name created with - * WT_CONNECTION::add_extractor., a string; default empty.} + * WT_CONNECTION::add_extractor., a string; default \c none.} * @config{format, the file format., a string\, chosen from the * following options: \c "btree"; default \c btree.} * @config{huffman_key, configure Huffman encoding for keys. Permitted * values are \c "none"\, \c "english"\, \c "utf8<file>" or \c * "utf16<file>". See @ref huffman for more information., a string; - * default empty.} + * default \c none.} * @config{huffman_value, configure Huffman encoding for values. * Permitted values are \c "none"\, \c "english"\, \c "utf8<file>" or \c * "utf16<file>". See @ref huffman for more information., a string; - * default empty.} + * default \c none.} * @config{immutable, configure the index to be immutable - that is an * index is not changed by any update to a record in the table., a * boolean flag; default \c false.} @@ -1600,8 +1600,8 @@ struct __wt_connection { * prepended to each log record\, may contain strftime conversion * specifications., a string; default \c "%b %d %H:%M:%S".} * @config{ wait, seconds to wait between each - * write of the log records., an integer between 0 and 100000; default - * \c 0.} + * write of the log records; setting this value above 0 configures + * statistics logging., an integer between 0 and 100000; default \c 0.} * @config{ ),,} * @config{verbose, enable messages for various events. Only available * if WiredTiger is configured with --enable-verbose. Options are given @@ -1905,7 +1905,7 @@ struct __wt_connection { * records. Permitted values are \c "none" or \c "bzip2"\, \c "snappy" or * custom compression engine \c "name" created with * WT_CONNECTION::add_compressor. See @ref compression for more information., a - * string; default empty.} + * string; default \c none.} * @config{ enabled, enable * logging subsystem., a boolean flag; default \c false.} * @config{ file_max, the maximum size of log files., an @@ -1979,9 +1979,10 @@ struct __wt_connection { * record\, may contain strftime conversion specifications., a string; default * \c "%b %d %H:%M:%S".} * @config{ wait, seconds to wait - * between each write of the log records., an integer between 0 and 100000; - * default \c 0.} - * @config{ ),,} + * between each write of the log records; setting this value above 0 configures + * statistics logging., an integer between 0 and 100000; default \c 0.} + * @config{ + * ),,} * @config{transaction_sync = (, how to sync log records when the transaction * commits., a set of related configuration options defined below.} * @config{ enabled, whether to sync the log on every @@ -2612,7 +2613,8 @@ struct __wt_collator { const char *uri, WT_CONFIG_ITEM *appcfg, WT_COLLATOR **customp); /*! - * If non-NULL, a callback performed when the database is closed. + * If non-NULL a callback performed when the data source is closed + * for customized extractors otherwise when the database is closed. * * The WT_COLLATOR::terminate callback is intended to allow cleanup, * the handle will not be subsequently accessed by WiredTiger. @@ -3036,7 +3038,9 @@ struct __wt_extractor { const char *uri, WT_CONFIG_ITEM *appcfg, WT_EXTRACTOR **customp); /*! - * If non-NULL, a callback performed when the database is closed. + * If non-NULL a callback performed when the index or column group + * is closed for customized extractors otherwise when the database + * is closed. * * The WT_EXTRACTOR::terminate callback is intended to allow cleanup, * the handle will not be subsequently accessed by WiredTiger. diff --git a/src/third_party/wiredtiger/src/lsm/lsm_worker.c b/src/third_party/wiredtiger/src/lsm/lsm_worker.c index 955dd09193b..703453305ba 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_worker.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_worker.c @@ -141,8 +141,10 @@ __lsm_worker(void *arg) ret = 0; } else if (ret == EBUSY) ret = 0; - /* Clear any state */ - WT_CLEAR_BTREE_IN_SESSION(session); + + /* Paranoia: clear session state. */ + session->dhandle = NULL; + __wt_lsm_manager_free_work_unit(session, entry); entry = NULL; progress = 1; diff --git a/src/third_party/wiredtiger/src/os_posix/os_getenv.c b/src/third_party/wiredtiger/src/os_posix/os_getenv.c new file mode 100644 index 00000000000..37fe2af8501 --- /dev/null +++ b/src/third_party/wiredtiger/src/os_posix/os_getenv.c @@ -0,0 +1,25 @@ +/*- + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_getenv -- + * Get a non-NULL, greater than zero-length environment variable. + */ +int +__wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) +{ + const char *temp; + + *envp = NULL; + + if (((temp = getenv(variable)) != NULL) && strlen(temp) > 0) + return (__wt_strdup(session, temp, envp)); + + return (WT_NOTFOUND); +} diff --git a/src/third_party/wiredtiger/src/os_win/os_getenv.c b/src/third_party/wiredtiger/src/os_win/os_getenv.c new file mode 100644 index 00000000000..8cc94abb524 --- /dev/null +++ b/src/third_party/wiredtiger/src/os_win/os_getenv.c @@ -0,0 +1,35 @@ +/*- + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_getenv -- + * Get a non-NULL, greater than zero-length environment variable. + */ +int +__wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) +{ + WT_DECL_RET; + DWORD size; + + *envp = NULL; + + size = GetEnvironmentVariableA(variable, NULL, 0); + if (size <= 1) + return (WT_NOTFOUND); + + WT_RET(__wt_calloc(session, 1, size, envp)); + + ret = GetEnvironmentVariableA(variable, *envp, size); + /* We expect the number of bytes not including nul terminator. */ + if ((ret + 1) != size) + WT_RET_MSG(session, __wt_errno(), + "GetEnvironmentVariableA failed: %s", variable); + + return (0); +} diff --git a/src/third_party/wiredtiger/src/os_win/os_snprintf.c b/src/third_party/wiredtiger/src/os_win/os_snprintf.c new file mode 100644 index 00000000000..70a11b84850 --- /dev/null +++ b/src/third_party/wiredtiger/src/os_win/os_snprintf.c @@ -0,0 +1,23 @@ +/*- + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +_Check_return_opt_ int __cdecl _wt_snprintf( + _Out_writes_(_MaxCount) char * _DstBuf, + _In_ size_t _MaxCount, + _In_z_ _Printf_format_string_ const char * _Format, ...) +{ + va_list args; + WT_DECL_RET; + + va_start(args, _Format); + ret = _wt_vsnprintf(_DstBuf, _MaxCount, _Format, args); + va_end(args); + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c b/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c index 1058203e326..055a45e5862 100644 --- a/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c +++ b/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c @@ -7,8 +7,6 @@ #include "wt_internal.h" -#undef vsnprintf - _Check_return_opt_ int __cdecl _wt_vsnprintf( _Out_writes_(_MaxCount) char * _DstBuf, _In_ size_t _MaxCount, @@ -17,12 +15,23 @@ _Check_return_opt_ int __cdecl _wt_vsnprintf( { int len; - len = (size_t)vsnprintf(_DstBuf, _MaxCount, _Format, _ArgList); + /* + * WiredTiger will call with length 0 to get the needed buffer size + * We call the count only version in this case since vsnprintf_s assumes + * length is greater than zero or else it triggers the invalid_parameter + * handler. + */ + if (_MaxCount == 0) { + return _vscprintf(_Format, _ArgList); + } + + len = (size_t)_vsnprintf_s( + _DstBuf, _MaxCount, _TRUNCATE, _Format, _ArgList); /* * The MSVC implementation returns -1 on truncation instead of what - * it would have written. We could iteratively grow the buffer, or - * just ask us how big a buffer they would like. + * it would have written. We could let callers iteratively grow the + * buffer, or just ask us how big a buffer they would like. */ if (len == -1) len = _vscprintf(_Format, _ArgList) + 1; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 839ab028afd..6ce8247c2d4 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -397,13 +397,15 @@ __wt_reconcile(WT_SESSION_IMPL *session, ret = __rec_col_fix(session, r, page); break; case WT_PAGE_COL_INT: - ret = __rec_col_int(session, r, page); + WT_WITH_PAGE_INDEX(session, + ret = __rec_col_int(session, r, page)); break; case WT_PAGE_COL_VAR: ret = __rec_col_var(session, r, page, salvage); break; case WT_PAGE_ROW_INT: - ret = __rec_row_int(session, r, page); + WT_WITH_PAGE_INDEX(session, + ret = __rec_row_int(session, r, page)); break; case WT_PAGE_ROW_LEAF: ret = __rec_row_leaf(session, r, page, salvage); diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index ca09d25a495..9a74b26ffec 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -165,8 +165,8 @@ __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_RET(__wt_meta_track_on(session)); - /* Be careful to ignore any btree handle in our caller. */ - WT_CLEAR_BTREE_IN_SESSION(session); + /* Paranoia: clear any handle from our caller. */ + session->dhandle = NULL; if (WT_PREFIX_MATCH(uri, "colgroup:")) ret = __drop_colgroup(session, uri, force, cfg); diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 02556eb942c..1fbc52821a6 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -139,9 +139,13 @@ __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX *idx) WT_DECL_RET; /* If there is a custom extractor configured, terminate it. */ - if (idx->extractor != NULL && idx->extractor->terminate != NULL) + if (idx->extractor != NULL && + idx->extractor_owned && idx->extractor->terminate != NULL) { WT_TRET(idx->extractor->terminate( idx->extractor, &session->iface)); + idx->extractor = NULL; + idx->extractor_owned = 0; + } __wt_free(session, idx->name); __wt_free(session, idx->source); diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 8f460dcc29f..0effc1a7248 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -547,9 +547,12 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, salvage, config, cfg); + /* Block out checkpoints to avoid spurious EBUSY errors. */ + __wt_spin_lock(session, &S2C(session)->checkpoint_lock); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_salvage, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_SALVAGE)); + __wt_spin_unlock(session, &S2C(session)->checkpoint_lock); err: API_END_RET_NOTFOUND_MAP(session, ret); } @@ -682,9 +685,12 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, upgrade, config, cfg); + /* Block out checkpoints to avoid spurious EBUSY errors. */ + __wt_spin_lock(session, &S2C(session)->checkpoint_lock); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_upgrade, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_UPGRADE)); + __wt_spin_unlock(session, &S2C(session)->checkpoint_lock); err: API_END_RET_NOTFOUND_MAP(session, ret); } @@ -702,9 +708,12 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, verify, config, cfg); + /* Block out checkpoints to avoid spurious EBUSY errors. */ + __wt_spin_lock(session, &S2C(session)->checkpoint_lock); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_verify, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY)); + __wt_spin_unlock(session, &S2C(session)->checkpoint_lock); err: API_END_RET_NOTFOUND_MAP(session, ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index f958361a905..d6aa483c18a 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -322,6 +322,7 @@ __wt_txn_release(WT_SESSION_IMPL *session) __wt_logrec_free(session, &txn->logrec); /* Discard any memory from the session's split stash that we can. */ + WT_ASSERT(session, session->split_gen == 0); if (session->split_stash_cnt > 0) __wt_split_stash_discard(session); |