diff options
95 files changed, 1691 insertions, 523 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 0ccdf59babc..df7a66238e8 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -294,6 +294,7 @@ env.Depends(wtdll, [filelistfile, version_file]) Default(wtlib, wtdll) wtbin = env.Program("wt", [ + "src/utilities/util_alter.c", "src/utilities/util_backup.c", "src/utilities/util_cpyright.c", "src/utilities/util_compact.c", diff --git a/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc b/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc index 0e666ac1dc0..97b8cd6dbdf 100644 --- a/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc +++ b/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc @@ -19,16 +19,19 @@ // under the License. // // ------------------------------------------------------------------- +#define __STDC_FORMAT_MACROS +#include <inttypes.h> +#include <errno.h> #include <limits.h> +#include <memory.h> #include <stdio.h> + #include <sys/ipc.h> #include <sys/shm.h> #include <sys/types.h> #include <sys/stat.h> #include <syslog.h> -#include <memory.h> -#include <errno.h> #ifndef STORAGE_LEVELDB_INCLUDE_PERF_COUNT_H_ #include "perf_count.h" @@ -36,8 +39,6 @@ #include "util/coding.h" -#define __STDC_FORMAT_MACROS -#include <inttypes.h> #ifdef OS_SOLARIS # include <atomic.h> diff --git a/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc b/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc index 95c82289e18..c12aedc7bad 100644 --- a/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc +++ b/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc @@ -27,6 +27,7 @@ #include "leveldb_wt.h" #include <errno.h> +#include <stdlib.h> #include <sstream> #include <sys/param.h> #include <sys/stat.h> @@ -143,7 +144,14 @@ class ReplayIteratorImpl : public ReplayIterator { } private: - void SeekTo(WT_LSN *lsn); + /* + * A log sequence number, representing a position in the transaction log. + */ + typedef struct { + uint32_t file; /* Log file number */ + unsigned long long offset; /* Log file offset */ + } LSN; + void SeekTo(LSN *lsn); // No copying allowed ReplayIteratorImpl(const ReplayIterator&) { } void operator=(const ReplayIterator&) { } @@ -151,7 +159,7 @@ class ReplayIteratorImpl : public ReplayIterator { Status status_; WT_CURSOR *cursor_; WT_ITEM key_, value_; - WT_LSN lsn_; + LSN lsn_; bool valid_; uint64_t txnid; uint32_t fileid, opcount, optype, rectype; @@ -201,7 +209,7 @@ ReplayIteratorImpl::Next() { void ReplayIteratorImpl::SeekToLast() { int ret = 0; - WT_LSN last_lsn; + LSN last_lsn; last_lsn.file = 0; if (cursor_ != NULL) { @@ -235,7 +243,7 @@ ReplayIteratorImpl::SeekToLast() { void ReplayIteratorImpl::SeekTo(const std::string& timestamp) { - WT_LSN target_lsn; + LSN target_lsn; int ret = 0; if (timestamp == "all") { @@ -260,7 +268,7 @@ ReplayIteratorImpl::SeekTo(const std::string& timestamp) { // Set the cursor on the first modification record at or after the // given LSN. void -ReplayIteratorImpl::SeekTo(WT_LSN *target_lsn) { +ReplayIteratorImpl::SeekTo(LSN *target_lsn) { int ret = 0; valid_ = false; diff --git a/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h b/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h index dc185183034..351eb9f3dda 100644 --- a/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h +++ b/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h @@ -43,6 +43,7 @@ #include "basho/perf_count.h" #endif +#include <pthread.h> #include "wiredtiger.h" #define WT_URI "table:data" diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh index 9968edc468a..fdf3b14e991 100755 --- a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh +++ b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh @@ -99,6 +99,11 @@ while test "$run" -le "$runmax"; do if test "$?" -ne "0"; then exit 1 fi + + # Copy the artifacts from the run + backup_dir=${home}_$(basename $wttest)_${run}_$(date +"%s") + rsync -r -m --include="*Stat*" --include="CONFIG.wtperf" --include="*monitor" --include="latency*" --include="test.stat" --exclude="*" $home/ $backup_dir + # Load is always using floating point, so handle separately l=`grep "^Load time:" ./WT_TEST/test.stat` if test "$?" -eq "0"; then diff --git a/src/third_party/wiredtiger/build_posix/Make.base b/src/third_party/wiredtiger/build_posix/Make.base index 5b945aca5e0..9354eb4b183 100644 --- a/src/third_party/wiredtiger/build_posix/Make.base +++ b/src/third_party/wiredtiger/build_posix/Make.base @@ -17,6 +17,7 @@ endif bin_PROGRAMS = wt wt_SOURCES =\ + src/utilities/util_alter.c \ src/utilities/util_backup.c \ src/utilities/util_cpyright.c \ src/utilities/util_compact.c \ diff --git a/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 b/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 new file mode 100644 index 00000000000..ca3639715e7 --- /dev/null +++ b/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 @@ -0,0 +1,74 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_COMPILE_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 4 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/src/third_party/wiredtiger/build_posix/configure.ac.in b/src/third_party/wiredtiger/build_posix/configure.ac.in index ad00b19a3bb..b7c39b5da8b 100644 --- a/src/third_party/wiredtiger/build_posix/configure.ac.in +++ b/src/third_party/wiredtiger/build_posix/configure.ac.in @@ -22,33 +22,12 @@ AC_PROG_CC(cc gcc) AC_PROG_CXX(c++ g++) AM_PROG_AS(as gas) -# Configure options. -AM_OPTIONS - define([AC_LIBTOOL_LANG_CXX_CONFIG], [:])dnl define([AC_LIBTOOL_LANG_F77_CONFIG], [:])dnl + LT_PREREQ(2.2.6) LT_INIT([pic-only]) AC_SUBST([LIBTOOL_DEPS]) - -# If enable-strict is configured, turn on as much error checking as we can for -# this compiler. Intended for developers, and only works for gcc/clang, but it -# fills a need. -if test "$wt_cv_enable_strict" = "yes"; then - wt_cv_cc_version="`$CC --version | sed -eq`" - case "$wt_cv_cc_version" in - *clang*) - AM_CLANG_WARNINGS($wt_cv_cc_version);; - *cc*|*CC*) # cc, CC, gcc, GCC - AM_GCC_WARNINGS($wt_cv_cc_version);; - *) - AC_MSG_ERROR( - [--enable-strict does not support "$wt_cv_cc_version".]);; - esac - - AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings" -fi - AM_CONDITIONAL([POSIX_HOST], [true]) AM_CONDITIONAL([WINDOWS_HOST], [false]) @@ -69,6 +48,7 @@ AS_CASE([$host_cpu], [aarch64*], [wt_cv_arm64="yes"], [wt_cv_arm64="no"]) AM_CONDITIONAL([ARM64_HOST], [test "$wt_cv_arm64" = "yes"]) +AS_CASE([$host_os], [*solaris*], [wt_cv_solaris="yes"], [wt_cv_solaris="no"]) # This is a workaround as part of WT-2459. Currently, clang (v3.7) does not # support compiling the ASM code we have to perform the CRC checks on PowerPC. @@ -82,9 +62,17 @@ if test "$wt_cv_powerpc" = "yes" -a "$CC" != "$CCAS"; then fi AC_SUBST(AM_LIBTOOLFLAGS) +# WiredTiger uses anonymous unions to pad structures. It's part of C11, but +# some compilers require -std=c11 to support them. Turn on that flag for any +# compiler that supports it, except for Solaris, where gcc -std=c11 makes +# some none-C11 prototypes unavailable. +if test "$wt_cv_solaris" = "no"; then + AX_CHECK_COMPILE_FLAG([-std=c11], [AM_CFLAGS="$AM_CFLAGS -std=c11"]) +fi + if test "$GCC" = "yes"; then # The Solaris gcc compiler gets the additional -pthreads flag. - if test "`uname -s`" = "SunOS"; then + if test "$wt_cv_solaris" = "yes"; then AM_CFLAGS="$AM_CFLAGS -pthreads" fi @@ -95,11 +83,35 @@ if test "$GCC" = "yes"; then fi else # The Solaris native compiler gets the additional -mt flag. - if test "`uname -s`" = "SunOS"; then + if test "$wt_cv_solaris" = "yes"; then AM_CFLAGS="$AM_CFLAGS -mt" fi fi +# Linux requires _GNU_SOURCE to be defined +AS_CASE([$host_os], [linux*], [AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE"]) + +# If enable-strict is configured, turn on as much error checking as we can for +# this compiler. Intended for developers, and only works for gcc/clang, but it +# fills a need. +if test "$wt_cv_enable_strict" = "yes"; then + wt_cv_cc_version="`$CC --version | sed -eq`" + case "$wt_cv_cc_version" in + *clang*) + AM_CLANG_WARNINGS($wt_cv_cc_version);; + *cc*|*CC*) # cc, CC, gcc, GCC + AM_GCC_WARNINGS($wt_cv_cc_version);; + *) + AC_MSG_ERROR( + [--enable-strict does not support "$wt_cv_cc_version".]);; + esac + + AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings" +fi + +# Configure options. +AM_OPTIONS + # Java and Python APIs if test "$wt_cv_enable_java" = "yes" -o "$wt_cv_enable_python" = "yes"; then # Only a warning, we need to build release packages without SWIG. @@ -121,7 +133,7 @@ if test "$wt_cv_enable_java" = "yes"; then fi if test "$wt_cv_enable_python" = "yes"; then - AM_PATH_PYTHON([2.6]) + AM_PATH_PYTHON([2.7]) if test -n "$with_python_prefix" ; then PYTHON_INSTALL_ARG="-d $with_python_prefix" fi @@ -159,11 +171,6 @@ if test "$ac_cv_sizeof_void_p" != "8" ; then fi AC_MSG_RESULT(yes) -# Linux requires _GNU_SOURCE to be defined -case "$host_os" in -linux*) AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE" ;; -esac - # Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. BUFFER_ALIGNMENT=0 if test "$ax_cv_func_posix_memalign_works" = "yes" ; then diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 2b7ef4a94e1..98f9b5a230a 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -118,8 +118,7 @@ lsm_config = [ ]), ] -# Per-file configuration -file_config = format_meta + [ +file_runtime_config = [ Config('access_pattern_hint', 'none', r''' It is recommended that workloads that consist primarily of updates and/or point queries specify \c random. Workloads that @@ -128,6 +127,14 @@ file_config = format_meta + [ option leads to an advisory call to an appropriate operating system API where available''', choices=['none', 'random', 'sequential']), + Config('cache_resident', 'false', r''' + do not ever evict the object's pages from cache. Not compatible with + LSM tables; see @ref tuning_cache_resident for more information''', + type='boolean'), +] + +# Per-file configuration +file_config = format_meta + file_runtime_config + [ Config('block_allocation', 'best', r''' configure block allocation. Permitted values are \c "first" or \c "best"; the \c "first" configuration uses a first-available @@ -146,10 +153,6 @@ file_config = format_meta + [ WT_CONNECTION::add_compressor. If WiredTiger has builtin support for \c "lz4", \c "snappy", \c "zlib" or \c "zstd" compression, these names are also available. See @ref compression for more information'''), - Config('cache_resident', 'false', r''' - do not ever evict the object's pages from cache. Not compatible with - LSM tables; see @ref tuning_cache_resident for more information''', - type='boolean'), Config('checksum', 'uncompressed', r''' configure block checksums; permitted values are <code>on</code> (checksum all blocks), <code>off</code> (checksum no blocks) and @@ -727,7 +730,7 @@ wiredtiger_open_common =\ Config('hazard_max', '1000', r''' maximum number of simultaneous hazard pointers per session handle''', - min='15'), + min=15, undoc=True), Config('mmap', 'true', r''' Use memory mapping to access files when possible''', type='boolean'), @@ -834,6 +837,8 @@ methods = { 'WT_CURSOR.reconfigure' : Method(cursor_runtime_config), +'WT_SESSION.alter' : Method(file_runtime_config), + 'WT_SESSION.close' : Method([]), 'WT_SESSION.compact' : Method([ diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index fe9a17b7799..13d67ef961b 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -159,6 +159,7 @@ src/packing/pack_impl.c src/packing/pack_stream.c src/reconcile/rec_track.c src/reconcile/rec_write.c +src/schema/schema_alter.c src/schema/schema_create.c src/schema/schema_drop.c src/schema/schema_list.c diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 372a251bf01..53a3df87615 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -1,5 +1,4 @@ # List of WiredTiger #defines that are "unused", but it's OK. -ALIGN_CHECK API_CALL API_CALL_NOCONF API_SESSION_INIT @@ -8,17 +7,18 @@ JOINABLE_CURSOR_CALL_CHECK LF_MASK LLONG_MAX LLONG_MIN -SIZE_CHECK TXN_API_CALL TXN_API_CALL_NOCONF TXN_API_END WIN32_LEAN_AND_MEAN +WT_ALIGN_CHECK WT_ATOMIC_CAS WT_ATOMIC_FUNC WT_BLOCK_DESC_SIZE WT_BLOCK_HEADER_SIZE WT_CACHE_LINE_ALIGNMENT -WT_COMPILER_TYPE_ALIGN +WT_CACHE_LINE_PAD_BEGIN +WT_CACHE_LINE_PAD_END WT_CONN_CHECK_PANIC WT_DEADLOCK WT_DEBUG_BYTE @@ -35,10 +35,12 @@ WT_LOG_SLOT_MAXBITS WT_LOG_SLOT_UNBUFFERED_ISSET WT_PACKED_STRUCT_BEGIN WT_PACKED_STRUCT_END +WT_PADDING_CHECK WT_READ_BARRIER WT_REF_SIZE WT_SESSION_LOCKED_CHECKPOINT WT_SESSION_LOCKED_TURTLE +WT_SIZE_CHECK WT_STATS_FIELD_TO_OFFSET WT_STATS_SLOT_ID WT_STAT_CONN_DECRV diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index 06111245fe2..025f6d4c7eb 100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -12,6 +12,7 @@ file_parse() sed -n \ -e '/^int$/b loop' \ -e '/^static int$/b loop' \ + -e '/^static inline int$/b loop' \ -e 'd' \ -e ': loop' \ -e 'H' \ @@ -35,6 +36,7 @@ func_ok() -e '/int __compact_uri_analyze$/d' \ -e '/int __config_parser_close$/d' \ -e '/int __curlog_reset$/d' \ + -e '/int __cursor_fix_implicit$/d' \ -e '/int __handle_close_default$/d' \ -e '/int __handle_progress_default$/d' \ -e '/int __im_file_close$/d' \ @@ -43,7 +45,7 @@ func_ok() -e '/int __im_file_sync$/d' \ -e '/int __im_fs_directory_list_free$/d' \ -e '/int __im_fs_exist$/d' \ - -e '/int __posix_file_close$/d' \ + -e '/int __page_write_gen_wrapped_check$/d' \ -e '/int __posix_terminate$/d' \ -e '/int __rec_destroy_session$/d' \ -e '/int __win_terminate$/d' \ @@ -60,11 +62,14 @@ func_ok() -e '/int __wt_lsm_manager_pop_entry$/d' \ -e '/int __wt_once$/d' \ -e '/int __wt_posix_directory_list_free$/d' \ + -e '/int __wt_spin_init$/d' \ + -e '/int __wt_spin_trylock$/d' \ -e '/int __wt_stat_connection_desc$/d' \ -e '/int __wt_stat_dsrc_desc$/d' \ -e '/int __wt_stat_join_desc$/d' \ -e '/int __wt_win_directory_list_free$/d' \ -e '/int bdb_compare_reverse$/d' \ + -e '/int copyout_val$/d' \ -e '/int csv_error$/d' \ -e '/int csv_terminate$/d' \ -e '/int demo_file_close$/d' \ @@ -109,13 +114,14 @@ func_ok() -e '/int zstd_terminate$/d' } -# Complain about functions which return an "int" but which don't return except -# at the end of the function. for f in `find bench ext src test -name '*.[ci]'`; do if expr "$f" : '.*/windows_shim.c' > /dev/null; then continue fi + # Complain about functions which return an "int" but which don't return + # except at the end of the function. + # # Turn each function into a single line, then discard the function's # final "return" call, then discard any function that still has some # form of return assignment or call. @@ -128,6 +134,9 @@ for f in `find bench ext src test -name '*.[ci]'`; do -e '/WT_ILLEGAL_VALUE[A-Z_]*(/d' \ -e '/WT_PANIC[A-Z_]*(/d' \ -e '/WT_RET[A-Z_]*(/d' \ + -e '/WT_SIZE_CHECK_PACK(/d' \ + -e '/WT_SIZE_CHECK_UNPACK(/d' \ + -e '/WT_SYSCALL(/d' \ -e '/WT_TRET(/d' \ -e '/[^a-z_]ret = /d' \ -e '/[^a-z_]return (/d' \ @@ -142,6 +151,28 @@ for f in `find bench ext src test -name '*.[ci]'`; do echo "Add false positives to the list in dist/s_void." echo "==============================================" } + + # Complain about functions which declare a "ret" value but never use it. + file_parse $f | + grep 'WT_DECL_RET' | + sed -e '/ret =/d' \ + -e '/API_END_RET/d' \ + -e '/WT_CURSOR_NEEDKEY/d' \ + -e '/WT_CURSOR_NEEDVALUE/d' \ + -e '/WT_ERR/d' \ + -e '/WT_ILLEGAL_VALUE_ERR/d' \ + -e '/WT_SYSCALL.*ret/d' \ + -e '/WT_TRET/d' \ + -e 's/^\([^(]*\).*/\1/' \ + -e 's/^ *//' > $t + test -s $t && { + echo "==============================================" + echo "$f:" + cat $t | sed 's/^/ /' + echo "Function declares ret without using it." + echo "Add false positives to the list in dist/s_void." + echo "==============================================" + } done exit 0 diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index bcf5201bd90..022810d5c49 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -367,6 +367,9 @@ connection_stats = [ ########################################## SessionStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), SessionStat('session_open', 'open session count', 'no_clear,no_scale'), + SessionStat('session_table_alter_fail', 'table alter failed calls', 'no_clear,no_scale'), + SessionStat('session_table_alter_skip', 'table alter unchanged and skipped', 'no_clear,no_scale'), + SessionStat('session_table_alter_success', 'table alter successful calls', 'no_clear,no_scale'), SessionStat('session_table_compact_fail', 'table compact failed calls', 'no_clear,no_scale'), SessionStat('session_table_compact_success', 'table compact successful calls', 'no_clear,no_scale'), SessionStat('session_table_create_fail', 'table create failed calls', 'no_clear,no_scale'), diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index ea646604a76..8a1533011b2 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -557,6 +557,12 @@ session_ops(WT_SESSION *session) /*! [Create a column-store table] */ ret = session->create(session, "table:mytable", "key_format=r,value_format=S"); + + /*! [Alter a table] */ + ret = session->alter(session, + "table:mytable", "access_pattern_hint=random"); + /*! [Alter a table] */ + /*! [Create a column-store table] */ ret = session->drop(session, "table:mytable", NULL); diff --git a/src/third_party/wiredtiger/examples/c/ex_data_source.c b/src/third_party/wiredtiger/examples/c/ex_data_source.c index 6ed80dfcf19..387248f6ae2 100644 --- a/src/third_party/wiredtiger/examples/c/ex_data_source.c +++ b/src/third_party/wiredtiger/examples/c/ex_data_source.c @@ -46,6 +46,21 @@ my_data_source_init(WT_CONNECTION *connection) } /*! [WT_EXTENSION_API declaration] */ +/*! [WT_DATA_SOURCE alter] */ +static int +my_alter(WT_DATA_SOURCE *dsrc, WT_SESSION *session, + const char *uri, WT_CONFIG_ARG *config) +/*! [WT_DATA_SOURCE alter] */ +{ + /* Unused parameters */ + (void)dsrc; + (void)session; + (void)uri; + (void)config; + + return (0); +} + /*! [WT_DATA_SOURCE create] */ static int my_create(WT_DATA_SOURCE *dsrc, WT_SESSION *session, @@ -604,6 +619,7 @@ main(void) { /*! [WT_DATA_SOURCE register] */ static WT_DATA_SOURCE my_dsrc = { + my_alter, my_create, my_compact, my_drop, diff --git a/src/third_party/wiredtiger/ext/datasources/helium/helium.c b/src/third_party/wiredtiger/ext/datasources/helium/helium.c index dff86bd73ac..c584141b00d 100644 --- a/src/third_party/wiredtiger/ext/datasources/helium/helium.c +++ b/src/third_party/wiredtiger/ext/datasources/helium/helium.c @@ -3329,6 +3329,7 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) * compile-time should the structure change underneath us. */ static const WT_DATA_SOURCE wtds = { + NULL, /* No session.alter */ helium_session_create, /* session.create */ NULL, /* No session.compaction */ helium_session_drop, /* session.drop */ diff --git a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c index 866cd0663ce..0791b077750 100644 --- a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c +++ b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c @@ -1016,9 +1016,10 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { /* * List of the WT_DATA_SOURCE methods -- it's static so it breaks at - * compile-time should the structure changes underneath us. + * compile-time should the structure change underneath us. */ static WT_DATA_SOURCE wtds = { + NULL, /* No session.alter */ kvs_session_create, /* session.create */ NULL, /* No session.compaction */ kvs_session_drop, /* session.drop */ diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index e12e46b96e8..f6fbc77fee2 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "1b6c815a3fd34f14c20d5cd627155799d1de535c", + "commit": "8d2324943364286056ae399043f70b8a937de312", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-3.6" } diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i index 2264cb31ef1..3606bed1d69 100644 --- a/src/third_party/wiredtiger/lang/java/java_doc.i +++ b/src/third_party/wiredtiger/lang/java/java_doc.i @@ -27,6 +27,7 @@ COPYDOC(__wt_async_op, WT_ASYNC_OP, remove) COPYDOC(__wt_async_op, WT_ASYNC_OP, compact) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_id) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_type) +COPYDOC(__wt_session, WT_SESSION, alter) COPYDOC(__wt_session, WT_SESSION, close) COPYDOC(__wt_session, WT_SESSION, reconfigure) COPYDOC(__wt_session, WT_SESSION, open_cursor) diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i index d0910fa9f66..efc512f2f5a 100644 --- a/src/third_party/wiredtiger/lang/java/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i @@ -127,7 +127,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(javain) uint64_t "$javainput" %typemap(javaout) uint64_t { - return $jnicall; + return ($jnicall); } /* Return byte[] from cursor.get_value */ @@ -137,7 +137,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(javain) WT_ITEM, WT_ITEM * "$javainput" %typemap(javaout) WT_ITEM, WT_ITEM * { - return $jnicall; + return ($jnicall); } %typemap(in) WT_ITEM * (WT_ITEM item) %{ @@ -165,7 +165,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(out) int %{ if ($1 != 0 && $1 != WT_NOTFOUND) { throwWiredTigerException(jenv, $1); - return $null; + return ($null); } $result = $1; %} @@ -174,7 +174,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { if (!val) { SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, #name " is null"); - return $null; + return ($null); } %enddef @@ -537,7 +537,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; k.data = NULL; if ((ret = $self->get_key($self, &k)) != 0) throwWiredTigerException(jenv, ret); - return k; + return (k); } %javamethodmodifiers get_value_wrap "protected"; @@ -547,40 +547,40 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; v.data = NULL; if ((ret = $self->get_value($self, &v)) != 0) throwWiredTigerException(jenv, ret); - return v; + return (v); } %javamethodmodifiers insert_wrap "protected"; int insert_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->insert($self); + return ($self->insert($self)); } %javamethodmodifiers remove_wrap "protected"; int remove_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->remove($self); + return ($self->remove($self)); } %javamethodmodifiers search_wrap "protected"; int search_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->search($self); + return ($self->search($self)); } %javamethodmodifiers update_wrap "protected"; int update_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->update($self); + return ($self->update($self)); } %javamethodmodifiers _java_raw "protected"; bool _java_raw(JNIEnv *jenv) { (void)jenv; JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->c.lang_private; - return jcb->cursor_raw; + return (jcb->cursor_raw); } %javamethodmodifiers _java_init "protected"; @@ -617,7 +617,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; } protected static long getCPtr($javaclassname obj) { - return (obj == null) ? 0 : obj.swigCPtr; + return ((obj == null) ? 0 : obj.swigCPtr); } %} @@ -1129,7 +1129,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; k.data = NULL; if ((ret = $self->get_key($self, &k)) != 0) throwWiredTigerException(jenv, ret); - return k; + return (k); } %javamethodmodifiers get_value_wrap "protected"; @@ -1139,20 +1139,20 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; v.data = NULL; if ((ret = $self->get_value($self, &v)) != 0) throwWiredTigerException(jenv, ret); - return v; + return (v); } %javamethodmodifiers insert_wrap "protected"; int insert_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->insert($self); + return ($self->insert($self)); } %javamethodmodifiers remove_wrap "protected"; int remove_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->remove($self); + return ($self->remove($self)); } %javamethodmodifiers reset_wrap "protected"; @@ -1163,7 +1163,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; %javamethodmodifiers search_wrap "protected"; int search_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->search($self); + return ($self->search($self)); } %javamethodmodifiers search_near_wrap "protected"; @@ -1183,7 +1183,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int update_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->update($self); + return ($self->update($self)); } %javamethodmodifiers compare_wrap "protected"; @@ -1191,7 +1191,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int cmp, ret = $self->compare($self, other, &cmp); if (ret != 0) throwWiredTigerException(jenv, ret); - return cmp; + return (cmp); } %javamethodmodifiers equals_wrap "protected"; @@ -1199,14 +1199,14 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int cmp, ret = $self->equals($self, other, &cmp); if (ret != 0) throwWiredTigerException(jenv, ret); - return cmp; + return (cmp); } %javamethodmodifiers _java_raw "protected"; bool _java_raw(JNIEnv *jenv) { (void)jenv; JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->lang_private; - return jcb->cursor_raw; + return (jcb->cursor_raw); } %javamethodmodifiers _java_init "protected"; @@ -1955,7 +1955,7 @@ WT_CONNECTION *wiredtiger_open_wrap(JNIEnv *jenv, const char *home, const char * err: if (ret != 0) throwWiredTigerException(jenv, ret); - return conn; + return (conn); } } @@ -1985,7 +1985,7 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return asyncop; + return (asyncop); } } @@ -2009,7 +2009,7 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return session; + return (session); } } @@ -2037,14 +2037,14 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return cursor; + return (cursor); } } %extend __wt_async_op { long get_id_wrap(JNIEnv *jenv) { WT_UNUSED(jenv); - return (self->get_id(self)); + return ((long)self->get_id(self)); } } @@ -2055,6 +2055,6 @@ err: if (ret != 0) ret = self->transaction_pinned_range(self, &range); if (ret != 0) throwWiredTigerException(jenv, ret); - return range; + return ((long)range); } } diff --git a/src/third_party/wiredtiger/lang/python/setup.py b/src/third_party/wiredtiger/lang/python/setup.py index cf5b0c5cab1..9063a891fb9 100644 --- a/src/third_party/wiredtiger/lang/python/setup.py +++ b/src/third_party/wiredtiger/lang/python/setup.py @@ -35,8 +35,9 @@ from distutils.core import setup, Extension if not 'ARCHFLAGS' in os.environ: os.environ['ARCHFLAGS'] = '' -# Suppress warnings building SWIG generated code -extra_cflags = [ '-w', '-I../../src/include'] +# Suppress warnings building SWIG generated code. SWIG boiler plate +# functions have sign conversion warnings, so those warnings must be disabled. +extra_cflags = [ '-w', '-I../../src/include', '-Wno-sign-conversion'] dir = os.path.dirname(__file__) diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c index 8c074e503d9..6908802dbff 100644 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ b/src/third_party/wiredtiger/src/async/async_op.c @@ -254,7 +254,6 @@ __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) { WT_ASYNC *async; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; uint64_t cur_head, cur_tail, my_alloc, my_slot; #ifdef HAVE_DIAGNOSTIC WT_ASYNC_OP_IMPL *my_op; @@ -307,7 +306,7 @@ __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) WT_ORDERED_READ(cur_head, async->head); } WT_PUBLISH(async->head, my_alloc); - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 380493d1c0b..eb6647dd03c 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -61,7 +61,6 @@ __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) int __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) { - WT_DECL_RET; WT_EXT *ext; WT_EXTLIST *el; wt_off_t avail_eighty, avail_ninety, eighty, ninety; @@ -138,7 +137,7 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) __wt_spin_unlock(session, &block->live_lock); - return (ret); + return (0); } /* @@ -149,7 +148,6 @@ int __wt_block_compact_page_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool *skipp) { - WT_DECL_RET; WT_EXT *ext; WT_EXTLIST *el; wt_off_t limit, offset; @@ -194,7 +192,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, } #endif - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 3352b797fa9..d507cc0e396 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -1104,9 +1104,9 @@ __debug_cell(WT_DBG *ds, const WT_PAGE_HEADER *dsk, WT_CELL_UNPACK *unpack) case WT_CELL_VALUE_OVFL_RM: type = "ovfl"; addr: WT_RET(__wt_scr_alloc(session, 128, &buf)); - WT_RET(ds->f(ds, ", %s %s", type, + ret = ds->f(ds, ", %s %s", type, __wt_addr_string( - session, unpack->data, unpack->size, buf))); + session, unpack->data, unpack->size, buf)); __wt_scr_free(session, &buf); WT_RET(ret); break; diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index 7858d2cb14e..c2733d6567b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -44,7 +44,7 @@ __wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) * Wait for up to a second for hazard pointers to be cleared. */ for (hp = NULL, i = 0; i < 100; i++) { - if ((hp = __wt_page_hazard_check(session, ref)) == NULL) + if ((hp = __wt_hazard_check(session, ref)) == NULL) break; __wt_sleep(0, 10000); } diff --git a/src/third_party/wiredtiger/src/config/config_api.c b/src/third_party/wiredtiger/src/config/config_api.c index 76a51903588..05c5c1287a7 100644 --- a/src/third_party/wiredtiger/src/config/config_api.c +++ b/src/third_party/wiredtiger/src/config/config_api.c @@ -158,24 +158,25 @@ wiredtiger_config_validate(WT_SESSION *wt_session, } /* - * __conn_foc_add -- + * __wt_conn_foc_add -- * Add a new entry into the connection's free-on-close list. */ -static int -__conn_foc_add(WT_SESSION_IMPL *session, const void *p) +void +__wt_conn_foc_add(WT_SESSION_IMPL *session, const void *p) { WT_CONNECTION_IMPL *conn; conn = S2C(session); /* - * Our caller is expected to be holding any locks we need. + * Callers of this function are expected to be holding the connection's + * api_lock. + * + * All callers of this function currently ignore errors. */ - WT_RET(__wt_realloc_def( - session, &conn->foc_size, conn->foc_cnt + 1, &conn->foc)); - - conn->foc[conn->foc_cnt++] = (void *)p; - return (0); + if (__wt_realloc_def( + session, &conn->foc_size, conn->foc_cnt + 1, &conn->foc) == 0) + conn->foc[conn->foc_cnt++] = (void *)p; } /* @@ -328,12 +329,12 @@ __wt_configure_method(WT_SESSION_IMPL *session, * order to avoid freeing chunks of memory twice. Again, this isn't a * commonly used API and it shouldn't ever happen, just leak it. */ - (void)__conn_foc_add(session, entry->base); - (void)__conn_foc_add(session, entry); - (void)__conn_foc_add(session, checks); - (void)__conn_foc_add(session, newcheck->type); - (void)__conn_foc_add(session, newcheck->checks); - (void)__conn_foc_add(session, newcheck_name); + __wt_conn_foc_add(session, entry->base); + __wt_conn_foc_add(session, entry); + __wt_conn_foc_add(session, checks); + __wt_conn_foc_add(session, newcheck->type); + __wt_conn_foc_add(session, newcheck->checks); + __wt_conn_foc_add(session, newcheck_name); /* * Instead of using locks to protect configuration information, assume diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 9d886cbf0bd..e4fd7937a40 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -162,6 +162,14 @@ static const WT_CONFIG_CHECK confchk_WT_CURSOR_reconfigure[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, + { "cache_resident", "boolean", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_begin_transaction[] = { { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," @@ -1066,6 +1074,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "append=false,overwrite=true", confchk_WT_CURSOR_reconfigure, 2 }, + { "WT_SESSION.alter", + "access_pattern_hint=none,cache_resident=false", + confchk_WT_SESSION_alter, 2 + }, { "WT_SESSION.begin_transaction", "isolation=,name=,priority=0,snapshot=,sync=", confchk_WT_SESSION_begin_transaction, 5 diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 1bc4a501ce2..474b8bbad8a 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2309,9 +2309,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, } WT_ERR(__wt_verbose_config(session, cfg)); - WT_ERR(__wt_config_gets(session, cfg, "hazard_max", &cval)); - conn->hazard_max = (uint32_t)cval.val; - WT_ERR(__wt_config_gets(session, cfg, "session_max", &cval)); conn->session_size = (uint32_t)cval.val + WT_EXTRA_INTERNAL_SESSIONS; diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 3571cc60115..02182daa7dc 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -68,7 +68,6 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) session, &conn->hot_backup_lock, "hot backup")); WT_RET(__wt_calloc_def(session, WT_PAGE_LOCKS, &conn->page_lock)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, conn->page_lock); for (i = 0; i < WT_PAGE_LOCKS; ++i) WT_RET( __wt_spin_init(session, &conn->page_lock[i], "btree page")); diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 6454503d6cb..d4ace127bb2 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -30,7 +30,6 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) /* WT_SESSION_IMPL array. */ WT_RET(__wt_calloc(session, conn->session_size, sizeof(WT_SESSION_IMPL), &conn->sessions)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, conn->sessions); /* * Open the default session. We open this before starting service diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index eb5e15ae5c3..0ab992bc88c 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -281,33 +281,38 @@ __curindex_search_near(WT_CURSOR *cursor, int *exact) * (usually) doesn't contain the primary key, so it is just a prefix of * any matching index key. That said, if there is an exact match, we * want to find the first matching index entry and set exact equal to - * zero. Do a search_near, step to the next entry if we land on one - * that is too small, then check that the prefix matches. + * zero. + * + * Do a search_near, and if we find an entry that is too small, step to + * the next one. In the unlikely event of a search past the end of the + * tree, go back to the last key. */ __wt_cursor_set_raw_key(child, &cursor->key); WT_ERR(child->search_near(child, &cmp)); - if (cmp < 0) - WT_ERR(child->next(child)); + if (cmp < 0) { + if ((ret = child->next(child)) == WT_NOTFOUND) + ret = child->prev(child); + WT_ERR(ret); + } /* * We expect partial matches, and want the smallest record with a key * greater than or equal to the search key. * - * If the key we find is shorter than the search key, it can't possibly - * match. + * If the found key starts with the search key, we indicate a match by + * setting exact equal to zero. * - * The only way for the key to be exactly equal is if there is an index - * on the primary key, because otherwise the primary key columns will - * be appended to the index key, but we don't disallow that (odd) case. + * The compare function expects application-supplied keys to come first + * so we flip the sign of the result to match what callers expect. */ found_key = child->key; - if (found_key.size < cursor->key.size) - WT_ERR(WT_NOTFOUND); - found_key.size = cursor->key.size; + if (found_key.size > cursor->key.size) + found_key.size = cursor->key.size; WT_ERR(__wt_compare( session, cindex->index->collator, &cursor->key, &found_key, exact)); + *exact = -*exact; WT_ERR(__curindex_move(cindex)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 2fa2a207c8a..013a64ef2d5 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -1333,7 +1333,7 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, WT_ERR(__wt_scr_alloc(session, 0, &tmp)); if (columns != NULL) { WT_ERR(__wt_struct_reformat(session, table, - columns, strlen(columns), NULL, 1, tmp)); + columns, strlen(columns), NULL, false, tmp)); WT_ERR(__wt_strndup( session, tmp->data, tmp->size, &cursor->value_format)); WT_ERR(__wt_strdup(session, columns, &cjoin->projection)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index d590dbd828a..a0a3ffdd974 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -270,7 +270,6 @@ __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, bool iskey, va_list ap) { WT_CONFIG_ITEM *names; - WT_DECL_RET; size_t needed; char **json_bufp; @@ -288,7 +287,7 @@ __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, WT_RET(__json_struct_unpackv(session, buffer, size, fmt, names, (u_char *)*json_bufp, needed + 1, iskey, ap)); - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 466a929596e..fae7667e44f 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -924,7 +924,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, WT_ERR(__wt_scr_alloc(session, 0, &tmp)); if (columns != NULL) { WT_ERR(__wt_struct_reformat(session, table, - columns, strlen(columns), NULL, true, tmp)); + columns, strlen(columns), NULL, false, tmp)); WT_ERR(__wt_strndup( session, tmp->data, tmp->size, &cursor->value_format)); diff --git a/src/third_party/wiredtiger/src/docs/build-pydoc.sh b/src/third_party/wiredtiger/src/docs/build-pydoc.sh index 5e6e3635be5..aef88fd4c97 100755 --- a/src/third_party/wiredtiger/src/docs/build-pydoc.sh +++ b/src/third_party/wiredtiger/src/docs/build-pydoc.sh @@ -3,4 +3,4 @@ TOP=$DOCS/.. . $TOP/config.sh cd python -PYTHONPATH=../../lang/python/src:$THRIFT_HOME/lib/python2.6/site-packages pydoc -w wiredtiger +PYTHONPATH=../../lang/python/src:$THRIFT_HOME/lib/python2.7/site-packages pydoc -w wiredtiger diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index 0f5c56d25ce..5726a1d19a1 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -37,6 +37,39 @@ In general, commands that modify the database or tables will run recovery by default and commands that only read data will not run recovery. <hr> +@section util_alter wt alter +Alter a table. + +@subsection util_alter_synopsis Synopsis +<code>wt [-RVv] [-C config] [-E secretkey ] [-h directory] alter uri configuration ...</code> + +The \c uri and \c configuration pairs may be specified to the +\c alter command. These configuration pairs can be used to modify the +configuration values from those passed to the WT_SESSION::create +call. + +The \c uri part of the configuration pair should match only one of the +objects being altered, but may be a prefix of the object being matched. +For example, the following two sets of configuration pairs are +equivalent in the case of altering a single table named \c xxx. + +@code +table access_pattern_hint=sequential +table:xxx access_pattern_hint=sequential +@endcode + +It's an error, however, to specify a matching prefix that matches more +than a single object being altered. + +Multiple \c configuration arguments may be specified. For example, the +following two sets of configuration pairs are equivalent: + +@code +table:xxx access_pattern_hint=random,cache_resident=false +table:xxx access_pattern_hint=random table:xxx cache_resident=false +@endcode + +<hr> @section util_backup wt backup Perform a backup of a database or set of data sources. diff --git a/src/third_party/wiredtiger/src/docs/testing.dox b/src/third_party/wiredtiger/src/docs/testing.dox index 7d454d54212..cf280e8f3ff 100644 --- a/src/third_party/wiredtiger/src/docs/testing.dox +++ b/src/third_party/wiredtiger/src/docs/testing.dox @@ -27,7 +27,7 @@ The WiredTiger unit test suite includes tests that cover: The WiredTiger Python test suite is built using the WiredTiger Python API and the Python unittest functionality (the test suite requires at -least Python version 2.6). +least Python version 2.7). The WiredTiger test suite automatically runs as part of every commit into the WiredTiger GitHub source tree. diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index 78d09a56ea9..1e0e2eaf99a 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,4 +1,19 @@ /*! @page upgrading Upgrading WiredTiger applications +@section version_291 Upgrading to Version 2.9.1 +<dl> +<dt>WiredTiger now requires Python 2.7 at minimum</dt> +<dd> +The minimum version of Python supported by WiredTiger is now 2.7 up from the +previous version of 2.6. This is due to extra unit tests added in this release +that depend on 2.7. This is not due to a change in the Python API. +</dd> + +<dt>Changes to hazard pointer configuration</dt> +<dd> +The \c hazard_max parameter to ::wiredtiger_open is now ignored. Memory is +allocated for hazard pointers as required by each session. +</dd> +</dl><hr> @section version_290 Upgrading to Version 2.9.0 <dl> diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 5b857566299..6fa728916de 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -1148,9 +1148,17 @@ retry: while (slot < max_entries) { !__wt_cache_aggressive(session)) continue; - /* Skip files if we have used all available hazard pointers. */ - if (btree->evict_ref == NULL && session->nhazard >= - conn->hazard_max - WT_MIN(conn->hazard_max / 2, 10)) + /* + * Skip files if we have too many active walks. + * + * This used to be limited by the configured maximum number of + * hazard pointers per session. Even though that ceiling has + * been removed, we need to test eviction with huge numbers of + * active trees before allowing larger numbers of hazard + * pointers in the walk session. + */ + if (btree->evict_ref == NULL && + session->nhazard > WT_EVICT_MAX_TREES) continue; /* diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 56a0fcfc790..5b17a78a4dd 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -37,7 +37,7 @@ __evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) * Check for a hazard pointer indicating another thread is using the * page, meaning the page cannot be evicted. */ - if (__wt_page_hazard_check(session, ref) == NULL) + if (__wt_hazard_check(session, ref) == NULL) return (0); WT_STAT_DATA_INCR(session, cache_eviction_hazard); diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index e591209f39a..4f69c258621 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1437,53 +1437,6 @@ __wt_page_swap_func( } /* - * __wt_page_hazard_check -- - * Return if there's a hazard pointer to the page in the system. - */ -static inline WT_HAZARD * -__wt_page_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) -{ - WT_CONNECTION_IMPL *conn; - WT_HAZARD *hp; - WT_SESSION_IMPL *s; - uint32_t i, j, hazard_size, max, session_cnt; - - conn = S2C(session); - - /* - * No lock is required because the session array is fixed size, but it - * may contain inactive entries. We must review any active session - * that might contain a hazard pointer, so insert a barrier before - * reading the active session count. That way, no matter what sessions - * come or go, we'll check the slots for all of the sessions that could - * have been active when we started our check. - */ - WT_STAT_CONN_INCR(session, cache_hazard_checks); - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (s = conn->sessions, i = 0, j = 0, max = 0; - i < session_cnt; ++s, ++i) { - if (!s->active) - continue; - WT_ORDERED_READ(hazard_size, s->hazard_size); - if (s->hazard_size > max) { - max = s->hazard_size; - WT_STAT_CONN_SET(session, - cache_hazard_max, max); - } - for (hp = s->hazard; hp < s->hazard + hazard_size; ++hp) { - ++j; - if (hp->ref == ref) { - WT_STAT_CONN_INCRV(session, - cache_hazard_walks, j); - return (hp); - } - } - } - WT_STAT_CONN_INCRV(session, cache_hazard_walks, j); - return (NULL); -} - -/* * __wt_skip_choose_depth -- * Randomly choose a depth for a skiplist insert. */ diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 6ea13ff63ac..70f6169200d 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -16,6 +16,8 @@ #define WT_EVICT_WALK_BASE 300 /* Pages tracked across file visits */ #define WT_EVICT_WALK_INCR 100 /* Pages added each walk */ +#define WT_EVICT_MAX_TREES 1000 /* Maximum walk points */ + /* * WT_EVICT_ENTRY -- * Encapsulation of an eviction candidate. diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 486aa50e86c..f2746fc76d9 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -62,39 +62,40 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 10 #define WT_CONFIG_ENTRY_WT_CURSOR_close 11 #define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 12 -#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 13 -#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 14 -#define WT_CONFIG_ENTRY_WT_SESSION_close 15 -#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 16 -#define WT_CONFIG_ENTRY_WT_SESSION_compact 17 -#define WT_CONFIG_ENTRY_WT_SESSION_create 18 -#define WT_CONFIG_ENTRY_WT_SESSION_drop 19 -#define WT_CONFIG_ENTRY_WT_SESSION_join 20 -#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 21 -#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 22 -#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 23 -#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 24 -#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 25 -#define WT_CONFIG_ENTRY_WT_SESSION_rename 26 -#define WT_CONFIG_ENTRY_WT_SESSION_reset 27 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 28 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 29 -#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 30 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 31 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 32 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 33 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 34 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 35 -#define WT_CONFIG_ENTRY_colgroup_meta 36 -#define WT_CONFIG_ENTRY_file_config 37 -#define WT_CONFIG_ENTRY_file_meta 38 -#define WT_CONFIG_ENTRY_index_meta 39 -#define WT_CONFIG_ENTRY_lsm_meta 40 -#define WT_CONFIG_ENTRY_table_meta 41 -#define WT_CONFIG_ENTRY_wiredtiger_open 42 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 43 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 44 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 45 +#define WT_CONFIG_ENTRY_WT_SESSION_alter 13 +#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 14 +#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 15 +#define WT_CONFIG_ENTRY_WT_SESSION_close 16 +#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 17 +#define WT_CONFIG_ENTRY_WT_SESSION_compact 18 +#define WT_CONFIG_ENTRY_WT_SESSION_create 19 +#define WT_CONFIG_ENTRY_WT_SESSION_drop 20 +#define WT_CONFIG_ENTRY_WT_SESSION_join 21 +#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 22 +#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 23 +#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 24 +#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 25 +#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 26 +#define WT_CONFIG_ENTRY_WT_SESSION_rename 27 +#define WT_CONFIG_ENTRY_WT_SESSION_reset 28 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 29 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 30 +#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 31 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 32 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 33 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 34 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 35 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 36 +#define WT_CONFIG_ENTRY_colgroup_meta 37 +#define WT_CONFIG_ENTRY_file_config 38 +#define WT_CONFIG_ENTRY_file_meta 39 +#define WT_CONFIG_ENTRY_index_meta 40 +#define WT_CONFIG_ENTRY_lsm_meta 41 +#define WT_CONFIG_ENTRY_table_meta 42 +#define WT_CONFIG_ENTRY_wiredtiger_open 43 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 44 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 45 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 46 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 7d3d07a6abc..60ce5f55234 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -255,12 +255,6 @@ struct __wt_connection_impl { size_t session_scratch_max; /* Max scratch memory per session */ - /* - * WiredTiger allocates space for a fixed number of hazard pointers - * in each thread of control. - */ - uint32_t hazard_max; /* Hazard array size */ - WT_CACHE *cache; /* Page cache */ volatile uint64_t cache_size; /* Cache size (either statically configured or the current size diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 7dce310dabf..be042bcd6cb 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -218,6 +218,7 @@ extern int __wt_config_getones_none(WT_SESSION_IMPL *session, const char *config extern int __wt_config_gets_def(WT_SESSION_IMPL *session, const char **cfg, const char *key, int def, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_config_subgetraw(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_config_subgets(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern void __wt_conn_foc_add(WT_SESSION_IMPL *session, const void *p) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); @@ -422,7 +423,7 @@ extern int __wt_lsm_manager_push_entry(WT_SESSION_IMPL *session, uint32_t type, extern int __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char *newconfig) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_curstat_lsm_init( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_tree_bloom_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); @@ -436,6 +437,7 @@ extern void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tre extern void __wt_lsm_tree_throttle( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool decrease_only) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_lsm_tree_alter( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_tree_drop( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_tree_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_lsm_tree_truncate( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); @@ -550,6 +552,7 @@ extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) extern int __wt_bulk_insert_fix( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_bulk_insert_fix_bitmap(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_bulk_insert_var( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_direct_io_size_check(WT_SESSION_IMPL *session, const char **cfg, const char *config_name, uint32_t *allocsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); @@ -654,6 +657,7 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp ) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern void __wt_hazard_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern void __wt_fill_hex(const uint8_t *src, size_t src_max, uint8_t *dest, size_t dest_max, size_t *lenp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); extern int __wt_raw_to_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index ce6afdd6e9c..22d78fc165a 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -10,8 +10,6 @@ #define WT_SIZET_FMT "zu" /* size_t format string */ /* Add GCC-specific attributes to types and function declarations. */ -#define WT_COMPILER_TYPE_ALIGN(x) __attribute__((aligned(x))) - #define WT_PACKED_STRUCT_BEGIN(name) \ struct __attribute__ ((__packed__)) name { #define WT_PACKED_STRUCT_END \ diff --git a/src/third_party/wiredtiger/src/include/hardware.h b/src/third_party/wiredtiger/src/include/hardware.h index 0e52818ae05..2530659db21 100644 --- a/src/third_party/wiredtiger/src/include/hardware.h +++ b/src/third_party/wiredtiger/src/include/hardware.h @@ -55,7 +55,19 @@ #else #define WT_CACHE_LINE_ALIGNMENT 64 #endif -#define WT_CACHE_LINE_ALIGNMENT_VERIFY(session, a) \ - WT_ASSERT(session, \ - WT_PTRDIFF(&(a)[1], &(a)[0]) >= WT_CACHE_LINE_ALIGNMENT && \ - WT_PTRDIFF(&(a)[1], &(a)[0]) % WT_CACHE_LINE_ALIGNMENT == 0) + +/* + * Pad a structure so an array of structures get separate cache lines. + * + * Note that we avoid compiler structure alignment because that requires + * allocating aligned blocks of memory, and alignment pollutes any other type + * that contains an aligned field. It is possible that a hot field positioned + * before this one will be on the same cache line, but not if it is also + * padded. + * + * This alignment has a small impact on portability as well, as we are using an + * anonymous union here which is supported under C11, earlier versions of + * the GNU standard, and MSVC versions as early as 2003. + */ +#define WT_CACHE_LINE_PAD_BEGIN union { struct { +#define WT_CACHE_LINE_PAD_END }; char __padding[WT_CACHE_LINE_ALIGNMENT]; }; diff --git a/src/third_party/wiredtiger/src/include/lint.h b/src/third_party/wiredtiger/src/include/lint.h index 1b64186cbab..e20a83144ee 100644 --- a/src/third_party/wiredtiger/src/include/lint.h +++ b/src/third_party/wiredtiger/src/include/lint.h @@ -9,8 +9,6 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ -#define WT_COMPILER_TYPE_ALIGN(x) - #define WT_PACKED_STRUCT_BEGIN(name) \ struct name { #define WT_PACKED_STRUCT_END \ diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index 6272b63b8a9..3f2cb2ba8e6 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -159,7 +159,8 @@ union __wt_lsn { !FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ WT_LOG_SLOT_JOINED(state) < WT_LOG_SLOT_BUF_MAX) -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_logslot { +struct __wt_logslot { + WT_CACHE_LINE_PAD_BEGIN volatile int64_t slot_state; /* Slot state */ int64_t slot_unbuffered; /* Unbuffered data in this slot */ int32_t slot_error; /* Error value */ @@ -176,6 +177,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_logslot { #define WT_SLOT_SYNC 0x04 /* Needs sync on release */ #define WT_SLOT_SYNC_DIR 0x08 /* Directory sync on release */ uint32_t flags; /* Flags */ + WT_CACHE_LINE_PAD_END }; #define WT_SLOT_INIT_FLAGS 0 diff --git a/src/third_party/wiredtiger/src/include/msvc.h b/src/third_party/wiredtiger/src/include/msvc.h index d5be5bd8c60..6c5c8b67647 100644 --- a/src/third_party/wiredtiger/src/include/msvc.h +++ b/src/third_party/wiredtiger/src/include/msvc.h @@ -19,8 +19,6 @@ /* * Add MSVC-specific attributes and pragmas to types and function declarations. */ -#define WT_COMPILER_TYPE_ALIGN(x) __declspec(align(x)) - #define WT_PACKED_STRUCT_BEGIN(name) \ __pragma(pack(push,1)) \ struct name { diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h index 84f015d6b67..6b81b1a6265 100644 --- a/src/third_party/wiredtiger/src/include/mutex.h +++ b/src/third_party/wiredtiger/src/include/mutex.h @@ -53,9 +53,11 @@ typedef union { /* Read/write lock */ * WiredTiger uses read/write locks for shared/exclusive access to resources. */ struct __wt_rwlock { + WT_CACHE_LINE_PAD_BEGIN const char *name; /* Lock name for debugging */ wt_rwlock_t rwlock; /* Read/write lock */ + WT_CACHE_LINE_PAD_END }; /* @@ -72,7 +74,8 @@ struct __wt_rwlock { #if SPINLOCK_TYPE == SPINLOCK_GCC -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { +struct __wt_spinlock { + WT_CACHE_LINE_PAD_BEGIN volatile int lock; /* @@ -84,13 +87,15 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { int16_t stat_count_off; /* acquisitions offset */ int16_t stat_app_usecs_off; /* waiting application threads offset */ int16_t stat_int_usecs_off; /* waiting server threads offset */ + WT_CACHE_LINE_PAD_END }; #elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX ||\ SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE ||\ SPINLOCK_TYPE == SPINLOCK_MSVC -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { +struct __wt_spinlock { + WT_CACHE_LINE_PAD_BEGIN wt_mutex_t lock; const char *name; /* Mutex name */ @@ -106,6 +111,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { int16_t stat_int_usecs_off; /* waiting server threads offset */ int8_t initialized; /* Lock initialized, for cleanup */ + WT_CACHE_LINE_PAD_END }; #else diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 76cb463602c..7dd523aea26 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -41,7 +41,7 @@ struct __wt_hazard { * WT_SESSION_IMPL -- * Implementation of WT_SESSION. */ -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { +struct __wt_session_impl { WT_SESSION iface; void *lang_private; /* Language specific private storage */ @@ -200,9 +200,13 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { #define WT_SESSION_FIRST_USE(s) \ ((s)->hazard == NULL) - /* The number of hazard pointers grows dynamically. */ -#define WT_HAZARD_INCR 1 - uint32_t hazard_size; /* Allocated slots in hazard array. */ + /* + * The hazard pointer array grows as necessary, initialize with 250 + * slots. + */ +#define WT_SESSION_INITIAL_HAZARD_SLOTS 250 + uint32_t hazard_size; /* Hazard pointer array slots */ + uint32_t hazard_inuse; /* Hazard pointer array slots in-use */ uint32_t nhazard; /* Count of active hazard pointers */ WT_HAZARD *hazard; /* Hazard pointer array */ }; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index e53414fc0c9..0daab83e166 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -445,6 +445,9 @@ struct __wt_connection_stats { int64_t rec_split_stashed_objects; int64_t session_cursor_open; int64_t session_open; + int64_t session_table_alter_fail; + int64_t session_table_alter_success; + int64_t session_table_alter_skip; int64_t session_table_compact_fail; int64_t session_table_compact_success; int64_t session_table_create_fail; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index afe5a1965c4..12fc2a0a5b7 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -67,10 +67,12 @@ struct __wt_named_snapshot { uint32_t snapshot_count; }; -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_txn_state { +struct __wt_txn_state { + WT_CACHE_LINE_PAD_BEGIN volatile uint64_t id; volatile uint64_t pinned_id; volatile uint64_t metadata_pinned; + WT_CACHE_LINE_PAD_END }; struct __wt_txn_global { diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 3e8dd45003e..0cc4a6f8439 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -62,7 +62,6 @@ __wt_txn_unmodify(WT_SESSION_IMPL *session) static inline int __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) { - WT_DECL_RET; WT_TXN_OP *op; WT_TXN *txn; @@ -77,7 +76,7 @@ __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_TXN_OP_INMEM : WT_TXN_OP_BASIC; op->u.upd = upd; upd->txnid = session->txn.id; - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/include/verify_build.h b/src/third_party/wiredtiger/src/include/verify_build.h index 477b9b7c134..8abc192892e 100644 --- a/src/third_party/wiredtiger/src/include/verify_build.h +++ b/src/third_party/wiredtiger/src/include/verify_build.h @@ -6,9 +6,6 @@ * See the file LICENSE for redistribution information. */ -#undef ALIGN_CHECK -#undef SIZE_CHECK - /* * NOTE: If you see a compile failure in this file, your compiler is laying out * structs in memory in a way WiredTiger does not expect. Please refer to the @@ -36,12 +33,12 @@ */ #define WT_STATIC_ASSERT(cond) (void)sizeof(char[1 - 2 * !(cond)]) -#define SIZE_CHECK(type, e) do { \ +#define WT_SIZE_CHECK(type, e) do { \ char __check_##type[1 - 2 * !(sizeof(type) == (e))]; \ (void)__check_##type; \ } while (0) -#define ALIGN_CHECK(type, a) \ +#define WT_ALIGN_CHECK(type, a) \ WT_STATIC_ASSERT(WT_ALIGN(sizeof(type), (a)) == sizeof(type)) /* @@ -53,8 +50,18 @@ static inline void __wt_verify_build(void) { /* Check specific structures weren't padded. */ - SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); - SIZE_CHECK(WT_REF, WT_REF_SIZE); + WT_SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); + WT_SIZE_CHECK(WT_REF, WT_REF_SIZE); + + /* Check specific structures were padded. */ +#define WT_PADDING_CHECK(s) \ + WT_STATIC_ASSERT( \ + sizeof(s) > WT_CACHE_LINE_ALIGNMENT || \ + sizeof(s) % WT_CACHE_LINE_ALIGNMENT == 0) + WT_PADDING_CHECK(WT_LOGSLOT); + WT_PADDING_CHECK(WT_RWLOCK); + WT_PADDING_CHECK(WT_SPINLOCK); + WT_PADDING_CHECK(WT_TXN_STATE); /* * The btree code encodes key/value pairs in size_t's, and requires at @@ -71,6 +78,3 @@ __wt_verify_build(void) */ WT_STATIC_ASSERT(sizeof(wt_off_t) == 8); } - -#undef ALIGN_CHECK -#undef SIZE_CHECK diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 2365135e08d..a6deed7e14e 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -800,6 +800,34 @@ struct __wt_session { #endif /*! + * Alter a table. + * + * This will allow modification of some table settings after + * creation. + * + * @snippet ex_all.c Alter a table + * + * @param session the session handle + * @param name the URI of the object to alter, such as \c "table:stock" + * @configstart{WT_SESSION.alter, see dist/api_data.py} + * @config{access_pattern_hint, It is recommended that workloads that + * consist primarily of updates and/or point queries specify \c random. + * Workloads that do many cursor scans through large ranges of data + * specify \c sequential and other workloads specify \c none. The + * option leads to an advisory call to an appropriate operating system + * API where available., a string\, chosen from the following options: + * \c "none"\, \c "random"\, \c "sequential"; default \c none.} + * @config{cache_resident, do not ever evict the object's pages from + * cache. Not compatible with LSM tables; see @ref + * tuning_cache_resident for more information., a boolean flag; default + * \c false.} + * @configend + * @errors + */ + int __F(alter)(WT_HANDLE_CLOSED(WT_SESSION) *session, + const char *name, const char *config); + + /*! * Close the session handle. * * This will release the resources associated with the session handle, @@ -2355,8 +2383,6 @@ struct __wt_connection { * in seconds at which to check for files that are inactive and close them., an * integer between 1 and 100000; default \c 10.} * @config{ ),,} - * @config{hazard_max, maximum number of simultaneous hazard pointers per - * session handle., an integer greater than or equal to 15; default \c 1000.} * @config{in_memory, keep data in-memory only. See @ref in_memory for more * information., a boolean flag; default \c false.} * @config{log = (, enable logging. Enabling logging uses three sessions from @@ -3439,6 +3465,14 @@ struct __wt_compressor { */ struct __wt_data_source { /*! + * Callback to alter an object. + * + * @snippet ex_data_source.c WT_DATA_SOURCE alter + */ + int (*alter)(WT_DATA_SOURCE *dsrc, WT_SESSION *session, + const char *uri, WT_CONFIG_ARG *config); + + /*! * Callback to create a new object. * * @snippet ex_data_source.c WT_DATA_SOURCE create @@ -4671,114 +4705,120 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_SESSION_CURSOR_OPEN 1186 /*! session: open session count */ #define WT_STAT_CONN_SESSION_OPEN 1187 +/*! session: table alter failed calls */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1188 +/*! session: table alter successful calls */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1189 +/*! session: table alter unchanged and skipped */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1190 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1188 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1191 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1189 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1192 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1190 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1193 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1191 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1194 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1192 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1195 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1193 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1196 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1194 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1197 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1195 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1198 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1196 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1199 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1197 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1200 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1198 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1201 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1199 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1202 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1200 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1203 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1201 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1204 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1202 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1205 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1203 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1206 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1204 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1207 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1205 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1208 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1206 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1209 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1207 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1210 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1208 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1211 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1209 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1212 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1210 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1213 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1211 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1214 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1212 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1215 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1213 +#define WT_STAT_CONN_PAGE_SLEEP 1216 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1214 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1217 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1215 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1218 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1216 +#define WT_STAT_CONN_TXN_BEGIN 1219 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1217 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1220 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1218 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1221 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1219 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1222 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1220 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1223 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1221 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1224 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1222 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1225 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1223 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1226 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1224 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1227 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1225 +#define WT_STAT_CONN_TXN_CHECKPOINT 1228 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1226 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1229 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1227 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1230 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1228 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1231 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1229 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1232 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1230 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1233 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1231 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1234 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1232 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1235 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1233 +#define WT_STAT_CONN_TXN_SYNC 1236 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1234 +#define WT_STAT_CONN_TXN_COMMIT 1237 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1235 +#define WT_STAT_CONN_TXN_ROLLBACK 1238 /*! * @} diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index 3ad6814a5d5..a29a34e5652 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -283,7 +283,6 @@ __wt_log_slot_init(WT_SESSION_IMPL *session) conn = S2C(session); log = conn->log; - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, log->slot_pool); for (i = 0; i < WT_SLOT_POOL; i++) log->slot_pool[i].slot_state = WT_LOG_SLOT_FREE; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index df209ce3b05..839648b97d7 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -409,13 +409,11 @@ static int __clsm_resize_chunks( WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm, u_int nchunks) { - WT_DECL_RET; WT_LSM_CURSOR_CHUNK *chunk; /* Don't allocate more iterators if we don't need them. */ - if (clsm->chunks_count >= nchunks) { - return (ret); - } + if (clsm->chunks_count >= nchunks) + return (0); WT_RET(__wt_realloc_def(session, &clsm->chunks_alloc, nchunks, &clsm->chunks)); @@ -423,7 +421,7 @@ __clsm_resize_chunks( WT_RET(__wt_calloc_one(session, &chunk)); clsm->chunks[clsm->chunks_count] = chunk; } - return (ret); + return (0); } /* @@ -434,9 +432,10 @@ static void __clsm_free_chunks(WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm) { size_t i; - for (i = 0; i < clsm->chunks_count; i++) { + + for (i = 0; i < clsm->chunks_count; i++) __wt_free(session, clsm->chunks[i]); - } + __wt_free(session, clsm->chunks); } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c index bae8206515e..7a6a40e380f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c @@ -45,7 +45,7 @@ __clsm_close_bulk(WT_CURSOR *cursor) total_chunks /= avg_chunks) ++chunk->generation; - WT_RET(__wt_lsm_meta_write(session, lsm_tree)); + WT_RET(__wt_lsm_meta_write(session, lsm_tree, NULL)); ++lsm_tree->dsk_gen; /* Close the LSM cursor */ diff --git a/src/third_party/wiredtiger/src/lsm/lsm_merge.c b/src/third_party/wiredtiger/src/lsm/lsm_merge.c index f05a9c4b2b7..ceb5f03a2f5 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_merge.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_merge.c @@ -579,7 +579,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) * Any errors that happened after the tree was locked are * fatal - we can't guarantee the state of the tree. */ - if ((ret = __wt_lsm_meta_write(session, lsm_tree)) != 0) + if ((ret = __wt_lsm_meta_write(session, lsm_tree, NULL)) != 0) WT_PANIC_ERR(session, ret, "Failed finalizing LSM merge"); lsm_tree->dsk_gen++; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_meta.c b/src/third_party/wiredtiger/src/lsm/lsm_meta.c index ec52af96231..46ead6d6ac4 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_meta.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_meta.c @@ -454,13 +454,14 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) * Write the metadata for an LSM tree. */ int -__wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +__wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, + const char *newconfig) { WT_DECL_ITEM(buf); WT_DECL_RET; WT_LSM_CHUNK *chunk; u_int i; - const char *new_cfg[] = { NULL, NULL, NULL }; + const char *new_cfg[] = { NULL, NULL, NULL, NULL, NULL }; char *new_metadata; bool first; @@ -504,8 +505,10 @@ __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_ERR(__wt_buf_catfmt(session, buf, "]")); /* Update the existing configuration with the new values. */ - new_cfg[0] = lsm_tree->config; - new_cfg[1] = buf->data; + new_cfg[0] = WT_CONFIG_BASE(session, lsm_meta); + new_cfg[1] = lsm_tree->config; + new_cfg[2] = buf->data; + new_cfg[3] = newconfig; WT_ERR(__wt_config_collapse(session, new_cfg, &new_metadata)); ret = __wt_metadata_update(session, lsm_tree->name, new_metadata); WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 80d20ad24f8..38d87dd852b 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -293,8 +293,6 @@ int __wt_lsm_tree_setup_bloom( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) { - WT_DECL_RET; - /* * The Bloom URI can be populated when the chunk is created, but * it isn't set yet on open or merge. @@ -302,8 +300,8 @@ __wt_lsm_tree_setup_bloom( if (chunk->bloom_uri == NULL) WT_RET(__wt_lsm_tree_bloom_name( session, lsm_tree, chunk->id, &chunk->bloom_uri)); - WT_RET(__lsm_tree_cleanup_old(session, chunk->bloom_uri)); - return (ret); + + return (__lsm_tree_cleanup_old(session, chunk->bloom_uri)); } /* @@ -758,7 +756,7 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) lsm_tree->chunk[lsm_tree->nchunks++] = chunk; WT_ERR(__wt_lsm_tree_setup_chunk(session, lsm_tree, chunk)); - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); lsm_tree->need_switch = false; ++lsm_tree->dsk_gen; @@ -843,6 +841,47 @@ __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, } /* + * __wt_lsm_tree_alter -- + * Alter an LSM tree. + */ +int +__wt_lsm_tree_alter( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DECL_RET; + WT_LSM_CHUNK *chunk; + WT_LSM_TREE *lsm_tree; + u_int i; + bool locked; + + locked = false; + + /* Get the LSM tree. */ + WT_WITH_HANDLE_LIST_LOCK(session, + ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)); + WT_RET(ret); + + /* Prevent any new opens. */ + __wt_lsm_tree_writelock(session, lsm_tree); + locked = true; + + /* Alter the chunks. */ + for (i = 0; i < lsm_tree->nchunks; i++) { + chunk = lsm_tree->chunk[i]; + WT_ERR(__wt_schema_alter(session, chunk->uri, cfg)); + if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) + WT_ERR( + __wt_schema_alter(session, chunk->bloom_uri, cfg)); + } + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, cfg[0])); + +err: if (locked) + __wt_lsm_tree_writeunlock(session, lsm_tree); + __wt_lsm_tree_release(session, lsm_tree); + return (ret); +} + +/* * __wt_lsm_tree_drop -- * Drop an LSM tree. */ @@ -955,7 +994,7 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session, } } - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); locked = false; __wt_lsm_tree_writeunlock(session, lsm_tree); WT_ERR(__wt_metadata_remove(session, olduri)); @@ -1010,7 +1049,7 @@ __wt_lsm_tree_truncate( WT_ERR(__wt_lsm_merge_update_tree( session, lsm_tree, 0, lsm_tree->nchunks, chunk)); - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); locked = false; __wt_lsm_tree_writeunlock(session, lsm_tree); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index f3414363e3e..d9c185a3f58 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -364,7 +364,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, /* Lock the tree, mark the chunk as on disk and update the metadata. */ __wt_lsm_tree_writelock(session, lsm_tree); F_SET(chunk, WT_LSM_CHUNK_ONDISK); - ret = __wt_lsm_meta_write(session, lsm_tree); + ret = __wt_lsm_meta_write(session, lsm_tree, NULL); ++lsm_tree->dsk_gen; /* Update the throttle time. */ @@ -469,7 +469,7 @@ __lsm_bloom_create(WT_SESSION_IMPL *session, /* Ensure the bloom filter is in the metadata. */ __wt_lsm_tree_writelock(session, lsm_tree); F_SET(chunk, WT_LSM_CHUNK_BLOOM); - ret = __wt_lsm_meta_write(session, lsm_tree); + ret = __wt_lsm_meta_write(session, lsm_tree, NULL); ++lsm_tree->dsk_gen; __wt_lsm_tree_writeunlock(session, lsm_tree); @@ -659,7 +659,7 @@ __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) err: /* Flush the metadata unless the system is in panic */ if (flush_metadata && ret != WT_PANIC) { __wt_lsm_tree_writelock(session, lsm_tree); - WT_TRET(__wt_lsm_meta_write(session, lsm_tree)); + WT_TRET(__wt_lsm_meta_write(session, lsm_tree, NULL)); __wt_lsm_tree_writeunlock(session, lsm_tree); } __lsm_unpin_chunks(session, &cookie); diff --git a/src/third_party/wiredtiger/src/meta/meta_table.c b/src/third_party/wiredtiger/src/meta/meta_table.c index 71be2496678..4f60728b2d2 100644 --- a/src/third_party/wiredtiger/src/meta/meta_table.c +++ b/src/third_party/wiredtiger/src/meta/meta_table.c @@ -262,8 +262,17 @@ __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep) key, WT_META_TRACKING(session) ? "true" : "false", __metadata_turtle(key) ? "" : "not "); - if (__metadata_turtle(key)) - return (__wt_turtle_read(session, key, valuep)); + if (__metadata_turtle(key)) { + /* + * The returned value should only be set if ret is non-zero, but + * Coverity is convinced otherwise. The code path is used enough + * that Coverity complains a lot, add an error check to get some + * peace and quiet. + */ + if ((ret = __wt_turtle_read(session, key, valuep)) != 0) + __wt_free(session, *valuep); + return (ret); + } /* * All metadata reads are at read-uncommitted isolation. That's diff --git a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c index d4ee532a2af..be8b1abda31 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c @@ -107,10 +107,13 @@ __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) __wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name); /* - * Our callers are often setting flags to cause a thread to exit. Add - * a barrier to ensure the flags are seen by the threads. + * Our callers often set flags to cause a thread to exit. Add a barrier + * to ensure exit flags are seen by the sleeping threads, otherwise we + * can wake up a thread, it immediately goes back to sleep, and we'll + * hang. Use a full barrier (we may not write before waiting on thread + * join). */ - WT_WRITE_BARRIER(); + WT_FULL_BARRIER(); /* * Fast path if we are in (or can enter), a state where the next waiter diff --git a/src/third_party/wiredtiger/src/os_win/os_fs.c b/src/third_party/wiredtiger/src/os_win/os_fs.c index 6c74f2f411f..2f76fff04a5 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fs.c +++ b/src/third_party/wiredtiger/src/os_win/os_fs.c @@ -16,7 +16,6 @@ static int __win_fs_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, bool *existp) { - WT_DECL_RET; WT_DECL_ITEM(name_wide); WT_SESSION_IMPL *session; @@ -43,8 +42,8 @@ __win_fs_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, uint32_t flags) { DWORD windows_error; - WT_DECL_RET; WT_DECL_ITEM(name_wide); + WT_DECL_RET; WT_SESSION_IMPL *session; WT_UNUSED(file_system); diff --git a/src/third_party/wiredtiger/src/os_win/os_utf8.c b/src/third_party/wiredtiger/src/os_win/os_utf8.c index f7d11c24f03..ccd8321aecf 100644 --- a/src/third_party/wiredtiger/src/os_win/os_utf8.c +++ b/src/third_party/wiredtiger/src/os_win/os_utf8.c @@ -18,7 +18,6 @@ __wt_to_utf16_string( { DWORD windows_error; int bufferSize; - WT_DECL_RET; bufferSize = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0); windows_error = __wt_getlasterror(); @@ -55,7 +54,6 @@ __wt_to_utf8_string( { DWORD windows_error; int bufferSize; - WT_DECL_RET; bufferSize = WideCharToMultiByte( CP_UTF8, 0, wide, -1, NULL, 0, NULL, NULL); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_track.c b/src/third_party/wiredtiger/src/reconcile/rec_track.c index 10a64fdf116..3795b6e5ae8 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_track.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_track.c @@ -35,7 +35,6 @@ __ovfl_discard_verbose( { WT_CELL_UNPACK *unpack, _unpack; WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 512, &tmp)); @@ -50,7 +49,7 @@ __ovfl_discard_verbose( __wt_addr_string(session, unpack->data, unpack->size, tmp)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 @@ -83,7 +82,6 @@ static int __ovfl_discard_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) { WT_CELL **cellp; - WT_DECL_RET; WT_OVFL_TRACK *track; uint32_t i; @@ -101,7 +99,7 @@ __ovfl_discard_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) __wt_free(session, track->discard); track->discard_entries = track->discard_allocated = 0; - return (ret); + return (0); } /* @@ -170,7 +168,6 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, WT_PAGE *page, WT_OVFL_REUSE *reuse, const char *tag) { WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 64, &tmp)); @@ -188,7 +185,7 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, WT_MIN(reuse->value_size, 40), (char *)WT_OVFL_REUSE_VALUE(reuse)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 @@ -568,7 +565,6 @@ __ovfl_txnc_verbose(WT_SESSION_IMPL *session, WT_PAGE *page, WT_OVFL_TXNC *txnc, const char *tag) { WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 64, &tmp)); @@ -583,7 +579,7 @@ __ovfl_txnc_verbose(WT_SESSION_IMPL *session, WT_MIN(txnc->value_size, 40), (char *)WT_OVFL_TXNC_VALUE(txnc)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 86749eef2e1..e82f449a50d 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -5519,7 +5519,6 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) static int __rec_split_discard(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_DECL_RET; WT_PAGE_MODIFY *mod; WT_MULTI *multi; uint32_t i; @@ -5579,7 +5578,7 @@ __rec_split_discard(WT_SESSION_IMPL *session, WT_PAGE *page) break; } - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c new file mode 100644 index 00000000000..26d800aa98e --- /dev/null +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -0,0 +1,180 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __alter_file -- + * Alter a file. + */ +static int +__alter_file(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) +{ + WT_DECL_RET; + const char *cfg[4], *filename; + char *config, *newconfig; + + filename = uri; + newconfig = NULL; + if (!WT_PREFIX_SKIP(filename, "file:")) + return (__wt_unexpected_object_type(session, uri, "file:")); + + /* Find the URI */ + WT_RET(__wt_metadata_search(session, uri, &config)); + + WT_ASSERT(session, newcfg[0] != NULL); + /* + * Start with the base configuration because collapse is like + * a projection and if we are reading older metadata, it may not + * have all the components. + */ + cfg[0] = WT_CONFIG_BASE(session, file_meta); + cfg[1] = config; + cfg[2] = newcfg[0]; + cfg[3] = NULL; + WT_ERR(__wt_config_collapse(session, cfg, &newconfig)); + /* + * Only rewrite if there are changes. + */ + if (strcmp(config, newconfig) != 0) + WT_ERR(__wt_metadata_update(session, uri, newconfig)); + else + WT_STAT_CONN_INCR(session, session_table_alter_skip); + +err: __wt_free(session, config); + __wt_free(session, newconfig); + return (ret); +} + +/* + * __alter_colgroup -- + * WT_SESSION::alter for a colgroup. + */ +static int +__alter_colgroup( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_COLGROUP *colgroup; + WT_DECL_RET; + + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); + + /* If we can get the colgroup, perform any potential alterations. */ + if ((ret = __wt_schema_get_colgroup( + session, uri, false, NULL, &colgroup)) == 0) + WT_TRET(__wt_schema_alter(session, colgroup->source, cfg)); + + return (ret); +} + +/* + * __alter_index -- + * WT_SESSION::alter for an index. + */ +static int +__alter_index( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_INDEX *idx; + WT_DECL_RET; + + /* If we can get the index, perform any potential alterations. */ + if ((ret = __wt_schema_get_index( + session, uri, false, NULL, &idx)) == 0) + WT_TRET(__wt_schema_alter(session, idx->source, cfg)); + + return (ret); +} + +/* + * __alter_table -- + * WT_SESSION::alter for a table. + */ +static int +__alter_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_COLGROUP *colgroup; + WT_DECL_RET; + WT_TABLE *table; + const char *name; + u_int i; + + name = uri; + (void)WT_PREFIX_SKIP(name, "table:"); + + WT_RET(__wt_schema_get_table( + session, name, strlen(name), true, &table)); + + /* + * Alter the column groups only if we are using the default + * column group. Otherwise the user should alter each + * index or column group explicitly. + */ + if (table->ncolgroups == 0) + for (i = 0; i < WT_COLGROUPS(table); i++) { + if ((colgroup = table->cgroups[i]) == NULL) + continue; + /* + * Alter the column group before updating the metadata + * to avoid the metadata for the table becoming + * inconsistent if we can't get exclusive access. + */ + WT_ERR(__wt_schema_alter( + session, colgroup->source, cfg)); + } +err: __wt_schema_release_table(session, table); + return (ret); +} + +/* + * __wt_schema_alter -- + * Process a WT_SESSION::alter operation for all supported types. + */ +int +__wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DATA_SOURCE *dsrc; + WT_DECL_RET; + + WT_RET(__wt_meta_track_on(session)); + + /* Paranoia: clear any handle from our caller. */ + session->dhandle = NULL; + + if (WT_PREFIX_MATCH(uri, "colgroup:")) + ret = __alter_colgroup(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "file:")) + ret = __alter_file(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "index:")) + ret = __alter_index(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "lsm:")) + ret = __wt_lsm_tree_alter(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "table:")) + ret = __alter_table(session, uri, cfg); + else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) + ret = dsrc->alter == NULL ? + __wt_object_unsupported(session, uri) : + dsrc->alter(dsrc, + &session->iface, uri, (WT_CONFIG_ARG *)cfg); + else + ret = __wt_bad_object_type(session, uri); + + /* + * Map WT_NOTFOUND to ENOENT, based on the assumption WT_NOTFOUND means + * there was no metadata entry. + */ + if (ret == WT_NOTFOUND) + ret = ENOENT; + + /* Bump the schema generation so that stale data is ignored. */ + ++S2C(session)->schema_gen; + + WT_TRET(__wt_meta_track_off(session, true, ret != 0)); + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index 65c955cf1e9..c1a4f257648 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -75,7 +75,7 @@ __drop_colgroup( /* * __drop_index -- - * WT_SESSION::drop for a colgroup. + * WT_SESSION::drop for an index. */ static int __drop_index( @@ -85,7 +85,7 @@ __drop_index( WT_DECL_RET; WT_TABLE *table; - /* If we can get the colgroup, detach it from the table. */ + /* If we can get the index, detach it from the table. */ if ((ret = __wt_schema_get_index( session, uri, force, &table, &idx)) == 0) { table->idx_complete = false; @@ -136,7 +136,7 @@ __drop_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) if ((idx = table->indices[i]) == NULL) continue; /* - * Drop the column group before updating the metadata to avoid + * Drop the index before updating the metadata to avoid * the metadata for the table becoming inconsistent if we can't * get exclusive access. */ diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 377bc0bdc2d..fe1bf821d3b 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -58,7 +58,6 @@ int __wt_session_copy_values(WT_SESSION_IMPL *session) { WT_CURSOR *cursor; - WT_DECL_RET; TAILQ_FOREACH(cursor, &session->cursors, q) if (F_ISSET(cursor, WT_CURSTD_VALUE_INT)) { @@ -80,7 +79,7 @@ __wt_session_copy_values(WT_SESSION_IMPL *session) F_SET(cursor, WT_CURSTD_VALUE_EXT); } - return (ret); + return (0); } /* @@ -130,9 +129,47 @@ __session_clear(WT_SESSION_IMPL *session) * For these reasons, be careful when clearing the session structure. */ memset(session, 0, WT_SESSION_CLEAR_SIZE(session)); - session->hazard_size = 0; - session->nhazard = 0; + WT_INIT_LSN(&session->bg_sync_lsn); + + session->hazard_inuse = 0; + session->nhazard = 0; +} + +/* + * __session_alter -- + * Alter a table setting. + */ +static int +__session_alter(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, alter, config, cfg); + + /* Disallow objects in the WiredTiger name space. */ + WT_ERR(__wt_str_name_check(session, uri)); + + /* + * We replace the default configuration listing with the current + * configuration. Otherwise the defaults for values that can be + * altered would override settings used by the user in create. + */ + cfg[0] = cfg[1]; + cfg[1] = NULL; + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_LOCK(session, + ret = __wt_schema_alter(session, uri, cfg)))); + +err: if (ret != 0) + WT_STAT_CONN_INCR(session, session_table_alter_fail); + else + WT_STAT_CONN_INCR(session, session_table_alter_success); + API_END_RET_NOTFOUND_MAP(session, ret); } /* @@ -1690,6 +1727,7 @@ __open_session(WT_CONNECTION_IMPL *conn, static const WT_SESSION stds = { NULL, NULL, + __session_alter, __session_close, __session_reconfigure, __wt_session_strerror, @@ -1717,6 +1755,7 @@ __open_session(WT_CONNECTION_IMPL *conn, }, stds_readonly = { NULL, NULL, + __session_alter, __session_close, __session_reconfigure, __wt_session_strerror, @@ -1820,17 +1859,13 @@ __open_session(WT_CONNECTION_IMPL *conn, * session close because access to it isn't serialized. Allocate the * first time we open this session. */ - if (WT_SESSION_FIRST_USE(session_ret)) - WT_ERR(__wt_calloc_def( - session, conn->hazard_max, &session_ret->hazard)); - - /* - * Set an initial size for the hazard array. It will be grown as - * required up to hazard_max. The hazard_size is reset on close, since - * __wt_hazard_close ensures the array is cleared - so it is safe to - * reset the starting size on each open. - */ - session_ret->hazard_size = 0; + if (WT_SESSION_FIRST_USE(session_ret)) { + WT_ERR(__wt_calloc_def(session, + WT_SESSION_INITIAL_HAZARD_SLOTS, &session_ret->hazard)); + session_ret->hazard_size = WT_SESSION_INITIAL_HAZARD_SLOTS; + session_ret->hazard_inuse = 0; + session_ret->nhazard = 0; + } /* Cache the offset of this session's statistics bucket. */ session_ret->stat_bucket = WT_STATS_SLOT_ID(session); diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index b0fa7e129bb..7e88ad183fe 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -13,6 +13,48 @@ static void __hazard_dump(WT_SESSION_IMPL *); #endif /* + * hazard_grow -- + * Grow a hazard pointer array. + */ +static int +hazard_grow(WT_SESSION_IMPL *session) +{ + WT_HAZARD *nhazard; + size_t size; + void *ohazard; + + /* + * Allocate a new, larger hazard pointer array and copy the contents of + * the original into place. + */ + size = session->hazard_size; + WT_RET(__wt_calloc_def(session, size * 2, &nhazard)); + memcpy(nhazard, session->hazard, size * sizeof(WT_HAZARD)); + + /* + * Swap the new hazard pointer array into place after initialization + * is complete (initialization must complete before eviction can see + * the new hazard pointer array), then schedule the original to be + * freed. + */ + ohazard = session->hazard; + WT_PUBLISH(session->hazard, nhazard); + + __wt_spin_lock(session, &S2C(session)->api_lock); + __wt_conn_foc_add(session, ohazard); + __wt_spin_unlock(session, &S2C(session)->api_lock); + + /* + * Increase the size of the session's pointer array after swapping it + * into place (the session's reference must be updated before eviction + * can see the new size). + */ + WT_PUBLISH(session->hazard_size, (uint32_t)(size * 2)); + + return (0); +} + +/* * __wt_hazard_set -- * Set a hazard pointer. */ @@ -23,17 +65,12 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp #endif ) { - WT_BTREE *btree; - WT_CONNECTION_IMPL *conn; WT_HAZARD *hp; - int restarts = 0; - btree = S2BT(session); - conn = S2C(session); *busyp = false; /* If a file can never be evicted, hazard pointers aren't required. */ - if (F_ISSET(btree, WT_BTREE_IN_MEMORY)) + if (F_ISSET(S2BT(session), WT_BTREE_IN_MEMORY)) return (0); /* @@ -46,6 +83,45 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp return (0); } + /* If we have filled the current hazard pointer array, grow it. */ + if (session->nhazard >= session->hazard_size) { + WT_ASSERT(session, + session->nhazard == session->hazard_size && + session->hazard_inuse == session->hazard_size); + WT_RET(hazard_grow(session)); + } + + /* + * If there are no available hazard pointer slots, make another one + * visible. + */ + if (session->nhazard >= session->hazard_inuse) { + WT_ASSERT(session, + session->nhazard == session->hazard_inuse && + session->hazard_inuse < session->hazard_size); + hp = &session->hazard[session->hazard_inuse++]; + } else { + WT_ASSERT(session, + session->nhazard < session->hazard_inuse && + session->hazard_inuse <= session->hazard_size); + + /* + * There must be an empty slot in the array, find it. Skip most + * of the active slots by starting after the active count slot; + * there may be a free slot before there, but checking is + * expensive. If we reach the end of the array, continue the + * search from the beginning of the array. + */ + for (hp = session->hazard + session->nhazard;; ++hp) { + if (hp >= session->hazard + session->hazard_inuse) + hp = session->hazard; + if (hp->ref == NULL) + break; + } + } + + WT_ASSERT(session, hp->ref == NULL); + /* * Do the dance: * @@ -59,82 +135,43 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp * pointer before it discards the page (the eviction server sets the * state to WT_REF_LOCKED, then flushes memory and checks the hazard * pointers). - * - * For sessions with many active hazard pointers, skip most of the - * active slots: there may be a free slot in there, but checking is - * expensive. Most hazard pointers are released quickly: optimize - * for that case. */ - for (hp = session->hazard + session->nhazard;; ++hp) { - /* - * If we get to the end of the array, either: - * 1. If we know there are free slots somewhere, and this is - * the first time through, continue the search from the - * start. Don't actually continue the loop because that - * will skip the first slot. - * 2. If we have searched all the way through and we have - * allocated the maximum number of slots, give up. - * 3. Allocate another increment of slots, up to the maximum. - * The slot we are on should now be available. - */ - if (hp >= session->hazard + session->hazard_size) { - if (session->nhazard < session->hazard_size && - restarts++ == 0) - hp = session->hazard; - else if (session->hazard_size >= conn->hazard_max) - break; - else - WT_PUBLISH(session->hazard_size, WT_MIN( - session->hazard_size + WT_HAZARD_INCR, - conn->hazard_max)); - } - - if (hp->ref != NULL) - continue; - - hp->ref = ref; + hp->ref = ref; #ifdef HAVE_DIAGNOSTIC - hp->file = file; - hp->line = line; + hp->file = file; + hp->line = line; #endif - /* Publish the hazard pointer before reading page's state. */ - WT_FULL_BARRIER(); + /* Publish the hazard pointer before reading page's state. */ + WT_FULL_BARRIER(); - /* - * Check if the page state is still valid, where valid means a - * state of WT_REF_MEM. - */ - if (ref->state == WT_REF_MEM) { - ++session->nhazard; - - /* - * Callers require a barrier here so operations holding - * the hazard pointer see consistent data. - */ - WT_READ_BARRIER(); - return (0); - } + /* + * Check if the page state is still valid, where valid means a + * state of WT_REF_MEM. + */ + if (ref->state == WT_REF_MEM) { + ++session->nhazard; /* - * The page isn't available, it's being considered for eviction - * (or being evicted, for all we know). If the eviction server - * sees our hazard pointer before evicting the page, it will - * return the page to use, no harm done, if it doesn't, it will - * go ahead and complete the eviction. - * - * We don't bother publishing this update: the worst case is we - * prevent some random page from being evicted. + * Callers require a barrier here so operations holding + * the hazard pointer see consistent data. */ - hp->ref = NULL; - *busyp = true; + WT_READ_BARRIER(); return (0); } -#ifdef HAVE_DIAGNOSTIC - __hazard_dump(session); -#endif - WT_RET_MSG(session, ENOMEM, - "session %p: hazard pointer table full", (void *)session); + /* + * The page isn't available, it's being considered for eviction + * (or being evicted, for all we know). If the eviction server + * sees our hazard pointer before evicting the page, it will + * return the page to use, no harm done, if it doesn't, it will + * go ahead and complete the eviction. + * + * We don't bother publishing this update: the worst case is we + * prevent some random page from being evicted. + */ + hp->ref = NULL; + *busyp = true; + return (0); } /* @@ -144,20 +181,17 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) { - WT_BTREE *btree; WT_HAZARD *hp; - btree = S2BT(session); - /* If a file can never be evicted, hazard pointers aren't required. */ - if (F_ISSET(btree, WT_BTREE_IN_MEMORY)) + if (F_ISSET(S2BT(session), WT_BTREE_IN_MEMORY)) return (0); /* * Clear the caller's hazard pointer. * The common pattern is LIFO, so do a reverse search. */ - for (hp = session->hazard + session->hazard_size - 1; + for (hp = session->hazard + session->hazard_inuse - 1; hp >= session->hazard; --hp) if (hp->ref == ref) { @@ -174,9 +208,13 @@ __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) /* * If this was the last hazard pointer in the session, * reset the size so that checks can skip this session. + * + * A write-barrier() is necessary before the change to + * the in-use value, the number of active references + * can never be less than the number of in-use slots. */ if (--session->nhazard == 0) - WT_PUBLISH(session->hazard_size, 0); + WT_PUBLISH(session->hazard_inuse, 0); return (0); } @@ -205,7 +243,7 @@ __wt_hazard_close(WT_SESSION_IMPL *session) * diagnostic. */ for (found = false, hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) + hp < session->hazard + session->hazard_inuse; ++hp) if (hp->ref != NULL) { found = true; break; @@ -233,7 +271,7 @@ __wt_hazard_close(WT_SESSION_IMPL *session) * can't think of a reason it would be). */ for (hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) + hp < session->hazard + session->hazard_inuse; ++hp) if (hp->ref != NULL) { hp->ref = NULL; --session->nhazard; @@ -247,6 +285,80 @@ __wt_hazard_close(WT_SESSION_IMPL *session) } /* + * hazard_get_reference -- + * Return a consistent reference to a hazard pointer array. + */ +static inline void +hazard_get_reference( + WT_SESSION_IMPL *session, WT_HAZARD **hazardp, uint32_t *hazard_inusep) +{ + /* + * Hazard pointer arrays can be swapped out from under us if they grow. + * First, read the current in-use value. The read must precede the read + * of the hazard pointer itself (so the in-use value is pessimistic + * should the hazard array grow), and additionally ensure we only read + * the in-use value once. Then, read the hazard pointer, also ensuring + * we only read it once. + * + * Use a barrier instead of marking the fields volatile because we don't + * want to slow down the rest of the hazard pointer functions that don't + * need special treatment. + */ + WT_ORDERED_READ(*hazard_inusep, session->hazard_inuse); + WT_ORDERED_READ(*hazardp, session->hazard); +} + +/* + * __wt_hazard_check -- + * Return if there's a hazard pointer to the page in the system. + */ +WT_HAZARD * +__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_CONNECTION_IMPL *conn; + WT_HAZARD *hp; + WT_SESSION_IMPL *s; + uint32_t i, j, hazard_inuse, max, session_cnt, walk_cnt; + + conn = S2C(session); + + WT_STAT_CONN_INCR(session, cache_hazard_checks); + + /* + * No lock is required because the session array is fixed size, but it + * may contain inactive entries. We must review any active session + * that might contain a hazard pointer, so insert a read barrier after + * reading the active session count. That way, no matter what sessions + * come or go, we'll check the slots for all of the sessions that could + * have been active when we started our check. + */ + WT_ORDERED_READ(session_cnt, conn->session_cnt); + for (s = conn->sessions, + i = j = max = walk_cnt = 0; i < session_cnt; ++s, ++i) { + if (!s->active) + continue; + + hazard_get_reference(s, &hp, &hazard_inuse); + + if (hazard_inuse > max) { + max = hazard_inuse; + WT_STAT_CONN_SET(session, cache_hazard_max, max); + } + + for (j = 0; j < hazard_inuse; ++hp, ++j) { + ++walk_cnt; + if (hp->ref == ref) { + WT_STAT_CONN_INCRV(session, + cache_hazard_walks, walk_cnt); + return (hp); + } + } + } + WT_STAT_CONN_INCRV(session, cache_hazard_walks, walk_cnt); + return (NULL); +} + +/* * __wt_hazard_count -- * Count how many hazard pointers this session has on the given page. */ @@ -254,11 +366,12 @@ u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref) { WT_HAZARD *hp; + uint32_t i, hazard_inuse; u_int count; - for (count = 0, hp = session->hazard + session->hazard_size - 1; - hp >= session->hazard; - --hp) + hazard_get_reference(session, &hp, &hazard_inuse); + + for (count = 0, i = 0; i < hazard_inuse; ++hp, ++i) if (hp->ref == ref) ++count; @@ -276,7 +389,7 @@ __hazard_dump(WT_SESSION_IMPL *session) WT_HAZARD *hp; for (hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) + hp < session->hazard + session->hazard_inuse; ++hp) if (hp->ref != NULL) __wt_errx(session, "session %p: hazard pointer %p: %s, line %d", diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 5acd9fc713f..a9c0b24ef29 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -808,6 +808,9 @@ static const char * const __stats_connection_desc[] = { "reconciliation: split objects currently awaiting free", "session: open cursor count", "session: open session count", + "session: table alter failed calls", + "session: table alter successful calls", + "session: table alter unchanged and skipped", "session: table compact failed calls", "session: table compact successful calls", "session: table create failed calls", @@ -1086,6 +1089,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing rec_split_stashed_objects */ /* not clearing session_cursor_open */ /* not clearing session_open */ + /* not clearing session_table_alter_fail */ + /* not clearing session_table_alter_success */ + /* not clearing session_table_alter_skip */ /* not clearing session_table_compact_fail */ /* not clearing session_table_compact_success */ /* not clearing session_table_create_fail */ @@ -1397,6 +1403,12 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, rec_split_stashed_objects); to->session_cursor_open += WT_STAT_READ(from, session_cursor_open); to->session_open += WT_STAT_READ(from, session_open); + to->session_table_alter_fail += + WT_STAT_READ(from, session_table_alter_fail); + to->session_table_alter_success += + WT_STAT_READ(from, session_table_alter_success); + to->session_table_alter_skip += + WT_STAT_READ(from, session_table_alter_skip); to->session_table_compact_fail += WT_STAT_READ(from, session_table_compact_fail); to->session_table_compact_success += diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index a70551cdeb2..26a0ed679e2 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -777,7 +777,6 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_calloc_def( session, conn->session_size, &txn_global->states)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, txn_global->states); for (i = 0, s = txn_global->states; i < conn->session_size; i++, s++) s->id = s->metadata_pinned = s->pinned_id = WT_TXN_NONE; diff --git a/src/third_party/wiredtiger/src/txn/txn_nsnap.c b/src/third_party/wiredtiger/src/txn/txn_nsnap.c index b12da9fbaea..65ec1a6662f 100644 --- a/src/third_party/wiredtiger/src/txn/txn_nsnap.c +++ b/src/third_party/wiredtiger/src/txn/txn_nsnap.c @@ -28,7 +28,6 @@ __nsnap_destroy(WT_SESSION_IMPL *session, WT_NAMED_SNAPSHOT *nsnap) static int __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) { - WT_DECL_RET; WT_NAMED_SNAPSHOT *found; WT_TXN_GLOBAL *txn_global; @@ -56,7 +55,7 @@ __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) __nsnap_destroy(session, found); WT_STAT_CONN_INCR(session, txn_snapshots_dropped); - return (ret); + return (0); } /* @@ -67,7 +66,6 @@ __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) static int __nsnap_drop_to(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name, bool inclusive) { - WT_DECL_RET; WT_NAMED_SNAPSHOT *last, *nsnap, *prev; WT_TXN_GLOBAL *txn_global; uint64_t new_nsnap_oldest; @@ -134,7 +132,7 @@ __nsnap_drop_to(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name, bool inclusive) new_nsnap_oldest == WT_TXN_NONE || !__wt_txn_visible_all(session, new_nsnap_oldest)); - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index c2cf6c22aa4..2658d877b63 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -26,6 +26,7 @@ extern int __wt_optopt; /* character checked for validity */ extern int __wt_optreset; /* reset getopt */ extern char *__wt_optarg; /* argument associated with option */ +int util_alter(WT_SESSION *, int, char *[]); int util_backup(WT_SESSION *, int, char *[]); int util_cerr(WT_CURSOR *, const char *, int); int util_compact(WT_SESSION *, int, char *[]); diff --git a/src/third_party/wiredtiger/src/utilities/util_alter.c b/src/third_party/wiredtiger/src/utilities/util_alter.c new file mode 100644 index 00000000000..d228c15cd48 --- /dev/null +++ b/src/third_party/wiredtiger/src/utilities/util_alter.c @@ -0,0 +1,50 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_alter(WT_SESSION *session, int argc, char *argv[]) +{ + WT_DECL_RET; + int ch; + char **configp; + + while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + + argc -= __wt_optind; + argv += __wt_optind; + + /* The remaining arguments are uri/string pairs. */ + if (argc % 2 != 0) + return (usage()); + + for (configp = argv; + configp != NULL && *configp != NULL; configp += 2) + if ((ret = session->alter( + session, configp[0], configp[1])) != 0) + break; + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "alter uri configuration ...\n", + progname, usage_prefix); + return (1); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 2054b94e3ce..1da56adf137 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -117,6 +117,10 @@ main(int argc, char *argv[]) func = NULL; switch (command[0]) { + case 'a': + if (strcmp(command, "alter") == 0) + func = util_alter; + break; case 'b': if (strcmp(command, "backup") == 0) func = util_backup; @@ -252,6 +256,7 @@ usage(void) "\t" "-v\t" "verbose\n"); fprintf(stderr, "commands:\n" + "\t" "alter\t alter an object\n" "\t" "backup\t database backup\n" "\t" "compact\t compact an object\n" "\t" "copyright copyright information\n" diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 9bfba3cd0df..e4f7af2e1b2 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -65,6 +65,10 @@ static CONFIG c[] = { "if timed run should drop core", /* 0% */ C_BOOL, 0, 0, 0, &g.c_abort, NULL }, + { "alter", + "if altering the table is enabled", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_alter, NULL }, + { "auto_throttle", "if LSM inserts are throttled", /* 90% */ C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL }, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 820bc020c9b..c1f4875dbb2 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -140,6 +140,7 @@ typedef struct { char *config_open; /* Command-line configuration */ uint32_t c_abort; /* Config values */ + uint32_t c_alter; uint32_t c_auto_throttle; uint32_t c_backups; uint32_t c_bitcnt; @@ -242,7 +243,7 @@ typedef struct { } GLOBAL; extern GLOBAL g; -typedef struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) { +typedef struct { WT_RAND_STATE rnd; /* thread RNG state */ uint64_t search; /* operations */ @@ -276,6 +277,7 @@ void bdb_remove(uint64_t, int *); void bdb_update(const void *, size_t, const void *, size_t); #endif +void *alter(void *); void *backup(void *); void *compact(void *); void config_clear(void); diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 283e2912daa..940318c87a9 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -50,10 +50,10 @@ static void print_item(const char *, WT_ITEM *); void wts_ops(int lastrun) { - TINFO *tinfo, total; + TINFO **tinfo_list, *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - pthread_t backup_tid, compact_tid, lrt_tid; + pthread_t alter_tid, backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; int running; @@ -61,6 +61,7 @@ wts_ops(int lastrun) conn = g.wts_conn; session = NULL; /* -Wconditional-uninitialized */ + memset(&alter_tid, 0, sizeof(alter_tid)); memset(&backup_tid, 0, sizeof(backup_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); memset(&lrt_tid, 0, sizeof(lrt_tid)); @@ -102,19 +103,24 @@ wts_ops(int lastrun) "=============== thread ops start ==============="); } - /* Create thread structure; start the worker threads. */ - tinfo = dcalloc((size_t)g.c_threads, sizeof(*tinfo)); + /* + * Create the per-thread structures and start the worker threads. + * Allocate the thread structures separately to minimize false sharing. + */ + tinfo_list = dcalloc((size_t)g.c_threads, sizeof(TINFO *)); for (i = 0; i < g.c_threads; ++i) { - tinfo[i].id = (int)i + 1; - tinfo[i].state = TINFO_RUNNING; - testutil_check( - pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); + tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); + tinfo->id = (int)i + 1; + tinfo->state = TINFO_RUNNING; + testutil_check(pthread_create(&tinfo->tid, NULL, ops, tinfo)); } /* * If a multi-threaded run, start optional backup, compaction and * long-running reader threads. */ + if (g.c_alter) + testutil_check(pthread_create(&alter_tid, NULL, alter, NULL)); if (g.c_backups) testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); if (g.c_compact) @@ -128,21 +134,22 @@ wts_ops(int lastrun) /* Clear out the totals each pass. */ memset(&total, 0, sizeof(total)); for (i = 0, running = 0; i < g.c_threads; ++i) { - total.commit += tinfo[i].commit; - total.deadlock += tinfo[i].deadlock; - total.insert += tinfo[i].insert; - total.remove += tinfo[i].remove; - total.rollback += tinfo[i].rollback; - total.search += tinfo[i].search; - total.update += tinfo[i].update; - - switch (tinfo[i].state) { + tinfo = tinfo_list[i]; + total.commit += tinfo->commit; + total.deadlock += tinfo->deadlock; + total.insert += tinfo->insert; + total.remove += tinfo->remove; + total.rollback += tinfo->rollback; + total.search += tinfo->search; + total.update += tinfo->update; + + switch (tinfo->state) { case TINFO_RUNNING: running = 1; break; case TINFO_COMPLETE: - tinfo[i].state = TINFO_JOINED; - (void)pthread_join(tinfo[i].tid, NULL); + tinfo->state = TINFO_JOINED; + (void)pthread_join(tinfo->tid, NULL); break; case TINFO_JOINED: break; @@ -154,7 +161,7 @@ wts_ops(int lastrun) */ if (fourths == 0 || (thread_ops != -1 && - tinfo[i].ops >= (uint64_t)thread_ops)) { + tinfo->ops >= (uint64_t)thread_ops)) { /* * On the last execution, optionally drop core * for recovery testing. @@ -163,7 +170,7 @@ wts_ops(int lastrun) static char *core = NULL; *core = 0; } - tinfo[i].quit = 1; + tinfo->quit = 1; } } track("ops", 0ULL, &total); @@ -173,10 +180,14 @@ wts_ops(int lastrun) if (fourths != -1) --fourths; } - free(tinfo); + for (i = 0; i < g.c_threads; ++i) + free(tinfo_list[i]); + free(tinfo_list); /* Wait for the backup, compaction, long-running reader threads. */ g.workers_finished = 1; + if (g.c_alter) + (void)pthread_join(alter_tid, NULL); if (g.c_backups) (void)pthread_join(backup_tid, NULL); if (g.c_compact) @@ -455,7 +466,7 @@ ops(void *arg) * 10% of the time, perform some read-only operations * from a checkpoint. * - * Skip that if we single-threaded and doing checks + * Skip that if we are single-threaded and doing checks * against a Berkeley DB database, because that won't * work because the Berkeley DB database records won't * match the checkpoint. Also skip if we are using diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index a709aa93a2e..b9788f1ac75 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -459,3 +459,47 @@ fclose_and_clear(FILE **fpp) testutil_die(errno, "fclose"); return; } + +/* + * alter -- + * Periodically alter a table's metadata. + */ +void * +alter(void *arg) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + u_int period; + bool access_value; + char buf[32]; + + (void)(arg); + conn = g.wts_conn; + + /* + * Only alter the access pattern hint. If we alter the cache resident + * setting we may end up with a setting that fills cache and doesn't + * allow it to be evicted. + */ + access_value = false; + + /* Open a session */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + while (!g.workers_finished) { + period = mmrand(NULL, 1, 10); + + snprintf(buf, sizeof(buf), + "access_pattern_hint=%s", access_value ? "random" : "none"); + access_value = !access_value; + if (session->alter(session, g.uri, buf) != 0) + break; + while (period > 0 && !g.workers_finished) { + --period; + sleep(1); + } + } + + testutil_check(session->close(session, NULL)); + return (NULL); +} diff --git a/src/third_party/wiredtiger/test/suite/test_alter01.py b/src/third_party/wiredtiger/test/suite/test_alter01.py new file mode 100644 index 00000000000..dfdf6b7a17e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_alter01.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_alter01.py +# Smoke-test the session alter operations. +class test_alter01(wttest.WiredTigerTestCase): + name = "alter01" + entries = 100 + # Settings for access_pattern_hint + types = [ + ('file', dict(uri='file:', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), + ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), + ('table-index', dict(uri='table:', use_cg=False, use_index=True)), + ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ] + hints = [ + ('default', dict(acreate='')), + ('none', dict(acreate='none')), + ('random', dict(acreate='random')), + ('sequential', dict(acreate='sequential')), + ] + access_alter=('', 'none', 'random', 'sequential') + # Settings for cache_resident + resid = [ + ('default', dict(ccreate='')), + ('false', dict(ccreate='false')), + ('true', dict(ccreate='true')), + ] + reopen = [ + ('no-reopen', dict(reopen=False)), + ('reopen', dict(reopen=True)), + ] + cache_alter=('', 'false', 'true') + scenarios = make_scenarios(types, hints, resid, reopen) + + def verify_metadata(self, metastr): + if metastr == '': + return + cursor = self.session.open_cursor('metadata:', None, None) + # + # Walk through all the metadata looking for the entries that are + # the file URIs for components of the table. + # + found = False + while True: + ret = cursor.next() + if ret != 0: + break + key = cursor.get_key() + check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \ + and key.find(self.name) != -1) + if check_meta: + value = cursor[key] + found = True + self.assertTrue(value.find(metastr) != -1) + cursor.close() + self.assertTrue(found == True) + + # Alter: Change the access pattern hint after creation + def test_alter01_access(self): + uri = self.uri + self.name + create_params = 'key_format=i,value_format=i,' + complex_params = '' + # + # If we're not explicitly setting the parameter, then don't + # modify create_params to test using the default. + # + if self.acreate != '': + access_param = 'access_pattern_hint=%s' % self.acreate + create_params += '%s,' % access_param + complex_params += '%s,' % access_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + access_param = 'access_pattern_hint=none' + if self.ccreate != '': + cache_param = 'cache_resident=%s' % self.ccreate + create_params += '%s,' % cache_param + complex_params += '%s,' % cache_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + cache_param = 'cache_resident=false' + + cgparam = '' + if self.use_cg or self.use_index: + cgparam = 'columns=(k,v),' + if self.use_cg: + cgparam += 'colgroups=(g0),' + + self.session.create(uri, create_params + cgparam) + # Add in column group or index settings. + if self.use_cg: + cgparam = 'columns=(v),' + suburi = 'colgroup:' + self.name + ':g0' + self.session.create(suburi, complex_params + cgparam) + if self.use_index: + suburi = 'index:' + self.name + ':i0' + self.session.create(suburi, complex_params + cgparam) + + # Put some data in table. + c = self.session.open_cursor(uri, None) + for k in range(self.entries): + c[k+1] = 1 + c.close() + + # Verify the string in the metadata + self.verify_metadata(access_param) + self.verify_metadata(cache_param) + + # Run through all combinations of the alter commands + # for all allowed settings. This tests having only one or + # the other set as well as having both set. It will also + # cover trying to change the setting to its current value. + for a in self.access_alter: + alter_param = '' + access_str = '' + if a != '': + access_str = 'access_pattern_hint=%s' % a + for c in self.cache_alter: + alter_param = '%s' % access_str + cache_str = '' + if c != '': + cache_str = 'cache_resident=%s' % c + alter_param += ',%s' % cache_str + if alter_param != '': + self.session.alter(uri, alter_param) + if self.reopen: + self.reopen_conn() + special = self.use_cg or self.use_index + if not special: + self.verify_metadata(access_str) + self.verify_metadata(cache_str) + else: + self.session.alter(suburi, alter_param) + self.verify_metadata(access_str) + self.verify_metadata(cache_str) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug011.py b/src/third_party/wiredtiger/test/suite/test_bug011.py index 29bb08ec2e5..969aaeb5b39 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug011.py +++ b/src/third_party/wiredtiger/test/suite/test_bug011.py @@ -30,25 +30,28 @@ import random, wiredtiger, wttest from wtdataset import SimpleDataSet # test_bug011.py -# Eviction working on more files than there are hazard pointers. +# Eviction working on more trees than the eviction server can walk +# simultaneously. There is a builtin limit of 1000 trees, we open double +# that, which makes this a long-running test. class test_bug011(wttest.WiredTigerTestCase): """ Test having eviction working on more files than the number of allocated hazard pointers. """ table_name = 'test_bug011' - ntables = 50 + ntables = 2000 nrows = 10000 nops = 10000 # Add connection configuration for this test. def conn_config(self, dir): - return 'cache_size=10MB,eviction_dirty_target=99,eviction_dirty_trigger=99,hazard_max=' + str(self.ntables / 2) + return 'cache_size=1GB' + @wttest.longtest("Eviction copes with lots of files") def test_eviction(self): cursors = [] datasets = [] for i in range(0, self.ntables): - this_uri = 'table:%s-%03d' % (self.table_name, i) + this_uri = 'table:%s-%05d' % (self.table_name, i) ds = SimpleDataSet(self, this_uri, self.nrows, config='allocation_size=1KB,leaf_page_max=1KB') ds.populate() @@ -57,9 +60,9 @@ class test_bug011(wttest.WiredTigerTestCase): # Switch over to on-disk trees with multiple leaf pages self.reopen_conn() - # Make sure we have a cursor for the table so it stays in cache. + # Make sure we have a cursor for every table so it stays in cache. for i in range(0, self.ntables): - this_uri = 'table:%s-%03d' % (self.table_name, i) + this_uri = 'table:%s-%05d' % (self.table_name, i) cursors.append(self.session.open_cursor(this_uri, None)) # Make use of the cache. diff --git a/src/third_party/wiredtiger/test/suite/test_config03.py b/src/third_party/wiredtiger/test/suite/test_config03.py index 88ca6ae3f39..6699f7d2650 100644 --- a/src/third_party/wiredtiger/test/suite/test_config03.py +++ b/src/third_party/wiredtiger/test/suite/test_config03.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!usr/bin/env python # # Public Domain 2014-2016 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. @@ -48,8 +48,6 @@ class test_config03(test_base03.test_base03): eviction_trigger_scenarios = wtscenario.quick_scenarios( 's_eviction_trigger', [50, 90, 95, 99], None) - hazard_max_scenarios = wtscenario.quick_scenarios('s_hazard_max', - [15, 50, 500], [0.4, 0.8, 0.8]) multiprocess_scenarios = wtscenario.quick_scenarios('s_multiprocess', [True,False], [1.0,1.0]) session_max_scenarios = wtscenario.quick_scenarios('s_session_max', @@ -66,13 +64,13 @@ class test_config03(test_base03.test_base03): verbose_scenarios = wtscenario.quick_scenarios('s_verbose', [None], None) config_vars = [ 'cache_size', 'create', 'error_prefix', 'eviction_target', - 'eviction_trigger', 'hazard_max', 'multiprocess', - 'session_max', 'verbose' ] + 'eviction_trigger', 'multiprocess', 'session_max', + 'verbose' ] scenarios = wtscenario.make_scenarios( cache_size_scenarios, create_scenarios, error_prefix_scenarios, eviction_target_scenarios, eviction_trigger_scenarios, - hazard_max_scenarios, multiprocess_scenarios, session_max_scenarios, + multiprocess_scenarios, session_max_scenarios, transactional_scenarios, verbose_scenarios, prune=1000) #wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \ diff --git a/src/third_party/wiredtiger/test/suite/test_config04.py b/src/third_party/wiredtiger/test/suite/test_config04.py index 204aa7e27d5..db8a5f4a16a 100644 --- a/src/third_party/wiredtiger/test/suite/test_config04.py +++ b/src/third_party/wiredtiger/test/suite/test_config04.py @@ -154,10 +154,6 @@ class test_config04(wttest.WiredTigerTestCase): 'eviction_trigger=86'), "/eviction target must be lower than the eviction trigger/") - def test_hazard_max(self): - # Note: There isn't any direct way to know that this was set. - self.common_test('hazard_max=50') - def test_invalid_config(self): msg = '/Unbalanced brackets/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_cursor10.py b/src/third_party/wiredtiger/test/suite/test_cursor10.py new file mode 100644 index 00000000000..b3cffeab4e9 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor10.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_cursor10.py +# Cursors with projections. +class test_cursor04(wttest.WiredTigerTestCase): + """ + Test cursor search and search_near + """ + table_name1 = 'test_cursor04' + nentries = 20 + + scenarios = make_scenarios([ + ('row', dict(key_format='S', uri='table')), + ('col', dict(key_format='r', uri='table')) + ]) + + def genkey(self, i): + if self.key_format == 'S': + return 'key' + str(i).zfill(5) # return key00001, key00002, etc. + else: + return long(i+1) + + def genvalue(self, i): + return [ 'v0:' + str(i), i+1, 'v2' + str(i+2), i+3 ] + + def extractkey(self, k): + if self.key_format == 'S': + return int(k[3:]) + else: + return long(k-1) + + def test_projection(self): + """ + Create entries, and read back in a regular and projected cursor + """ + tablearg = self.uri + ":" + self.table_name1 + create_args = 'columns=(k,v0,v1,v2,v3),value_format=SiSi,key_format=' \ + + self.key_format + self.session.create(tablearg, create_args) + + cursor = self.session.open_cursor(tablearg, None, None) + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + values = self.genvalue(i) + cursor.set_value(*values) + cursor.insert() + cursor.close() + cursor = self.session.open_cursor(tablearg, None, None) + count = 0 + for k,v0,v1,v2,v3 in cursor: + i = self.extractkey(k) + self.assertEqual(self.genkey(i), k) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() + cursor = self.session.open_cursor(tablearg + '(v3,v2,v1,v0,k)',\ + None, None) + count = 0 + for k1,v3,v2,v1,v0,k2 in cursor: + self.assertEqual(k1, k2) + i = self.extractkey(k1) + self.assertEqual(self.genkey(i), k1) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() + + def test_index_projection(self): + """ + Create entries, and read back in an index cursor with a projection + """ + tablearg = self.uri + ":" + self.table_name1 + indexarg = 'index:' + self.table_name1 + ':index1' + create_args = 'columns=(k,v0,v1,v2,v3),value_format=SiSi,key_format=' \ + + self.key_format + self.session.create(tablearg, create_args) + self.session.create(indexarg, 'columns=(v0,v2,v1,v3)') + cursor = self.session.open_cursor(tablearg, None, None) + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + values = self.genvalue(i) + cursor.set_value(*values) + cursor.insert() + cursor.close() + cursor = self.session.open_cursor(tablearg + '(v3,v2,v1,v0,k)',\ + None, None) + count = 0 + for k1,v3,v2,v1,v0,k2 in cursor: + self.assertEqual(k1, k2) + i = self.extractkey(k1) + self.assertEqual(self.genkey(i), k1) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hazard.py b/src/third_party/wiredtiger/test/suite/test_hazard.py new file mode 100644 index 00000000000..f2891fce526 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hazard.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_hazard.py +# Hazard pointer tests. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# Regression tests. +class test_hazard(wttest.WiredTigerTestCase): + + # Allocate a large number of hazard pointers in a session, forcing the + # hazard pointer array to repeatedly grow. + def test_hazard(self): + uri = "table:hazard" + ds = SimpleDataSet(self, uri, 1000) + ds.populate() + + # Open 10,000 cursors and pin a page to set a hazard pointer. + cursors = [] + for i in range(0, 10000): + c = self.session.open_cursor(uri, None) + c.set_key(ds.key(10)) + c.search() + cursors.append(c) + + # Close the cursors, clearing the hazard pointer. + for c in cursors: + c.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_index02.py b/src/third_party/wiredtiger/test/suite/test_index02.py index 9f39df003b1..4f424e5d3d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_index02.py +++ b/src/third_party/wiredtiger/test/suite/test_index02.py @@ -27,20 +27,26 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_index02.py # test search_near in indices class test_index02(wttest.WiredTigerTestCase): '''Test search_near in indices''' + scenarios = make_scenarios([ + ('index', dict(indexconfig='columns=(v)', ncol=1)), + ('index-with-key', dict(indexconfig='columns=(v,k)', ncol=2)), + ]) + basename = 'test_index02' tablename = 'table:' + basename indexname = 'index:' + basename + ":inverse" - def test_search_near(self): - '''Create a table, look for a nonexistent key''' + def test_search_near_exists(self): + '''Create a table, look for an existing key''' self.session.create(self.tablename, 'key_format=r,value_format=Q,columns=(k,v)') - self.session.create(self.indexname, 'columns=(v)') + self.session.create(self.indexname, self.indexconfig) cur = self.session.open_cursor(self.tablename, None, "append") cur.set_value(1) cur.insert() @@ -52,17 +58,60 @@ class test_index02(wttest.WiredTigerTestCase): cur.insert() cur.set_value(10) cur.insert() + cur.close() + + # Retry after reopening + for runs in xrange(2): + # search near should find a match + cur = self.session.open_cursor(self.indexname, None, None) + if self.ncol == 1: + cur.set_key(5) + else: + cur.set_key(5, 3) + self.assertEqual(cur.search_near(), 0) + + # Retry after reopening + self.reopen_conn() + + def test_search_near_between(self): + '''Create a table, look for a non-existing key''' + self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') + self.session.create(self.indexname, self.indexconfig) + cur = self.session.open_cursor(self.tablename) + for k in xrange(3): + cur[k] = 5 * k + 10 + cur.close() + + search_keys = [ 1, 11, 15, 19, 21 ] # search near should find a match - cur2 = self.session.open_cursor(self.indexname, None, None) - cur2.set_key(5) - self.assertEqual(cur2.search_near(), 0) + for runs in xrange(2): + cur = self.session.open_cursor(self.indexname, None, None) + for k in search_keys: + if self.ncol == 1: + cur.set_key(k) + else: + cur.set_key(k, 1) # [15,1] will completely match + exact = cur.search_near() + if self.ncol == 1: + found_key = cur.get_key() + else: + [ found_key, index ] = cur.get_key() + self.pr("search_near for " + str(k) + " found " + str(found_key) + " with exact " + str(exact)) + self.assertEqual(exact, cmp(found_key, k), "for key " + str(k)) + self.reopen_conn() - # Retry after reopening - self.reopen_conn() - cur3 = self.session.open_cursor(self.indexname, None, None) - cur3.set_key(5) - self.assertEqual(cur3.search_near(), 0) + def test_search_near_empty(self): + '''Create an empty table, look for a key''' + self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') + self.session.create(self.indexname, self.indexconfig) + + cur = self.session.open_cursor(self.indexname, None, None) + if self.ncol == 1: + cur.set_key(3) + else: + cur.set_key(3, 1) + self.assertEqual(cur.search_near(), wiredtiger.WT_NOTFOUND) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_join01.py b/src/third_party/wiredtiger/test/suite/test_join01.py index f3b13026896..2c4328dc7d3 100644 --- a/src/third_party/wiredtiger/test/suite/test_join01.py +++ b/src/third_party/wiredtiger/test/suite/test_join01.py @@ -110,8 +110,9 @@ class test_join01(wttest.WiredTigerTestCase): while jc.next() == 0: [k] = jc.get_keys() i = k - 1 - if do_proj: # our projection test simply reverses the values - [v2,v1,v0] = jc.get_values() + if do_proj: # our projection reverses the values and adds the key + [v2,v1,v0,kproj] = jc.get_values() + self.assertEquals(k, kproj) else: [v0,v1,v2] = jc.get_values() self.assertEquals(self.gen_values(i), [v0,v1,v2]) @@ -136,7 +137,7 @@ class test_join01(wttest.WiredTigerTestCase): if self.ref == 'index': expectstats.append('join: index:join01:index0: ' + statdesc) elif self.do_proj: - expectstats.append('join: table:join01(v2,v1,v0): ' + statdesc) + expectstats.append('join: table:join01(v2,v1,v0,k): ' + statdesc) else: expectstats.append('join: table:join01: ' + statdesc) self.check_stats(statcur, expectstats) @@ -228,7 +229,7 @@ class test_join01(wttest.WiredTigerTestCase): c.close() if do_proj: - proj_suffix = '(v2,v1,v0)' # Reversed values + proj_suffix = '(v2,v1,v0,k)' # Reversed values plus key else: proj_suffix = '' # Default projection (v0,v1,v2) diff --git a/src/third_party/wiredtiger/test/syscall/syscall.py b/src/third_party/wiredtiger/test/syscall/syscall.py index f97ea8a5749..59c2f347146 100644 --- a/src/third_party/wiredtiger/test/syscall/syscall.py +++ b/src/third_party/wiredtiger/test/syscall/syscall.py @@ -104,6 +104,9 @@ headpatterns = [ [ tracepat, 'trace_syscalls', 0], [ systempat, 'required_system', 0], [ runpat, 'run_args', 0] ] +pwrite_in = r'pwrite64' +pwrite_out = r'pwrite' + # To create breakpoints while debugging this script def bp(): import pdb @@ -416,8 +419,11 @@ class Runner: def call_compare(self, callname, result, eargs, errline): if callname in calls_returning_zero: return self.compare("EQ", result, "0", errline) - elif callname == 'pwrite': - return self.compare("EQ", result, eargs[2], errline) + elif callname == 'pwrite' or callname == 'pwrite64': + return self.compare("EQ", + re.sub(pwrite_in, pwrite_out, result), + re.sub(pwrite_in, pwrite_out, eargs[2]), + errline) else: self.fail(errline, 'call ' + callname + ': not known, use ASSERT_EQ()') @@ -469,13 +475,12 @@ class Runner: if not em: self.fail(errline, 'Unknown strace/dtruss output: ' + errline) return False - gotcall = em.groups()[0] + gotcall = re.sub(pwrite_in, pwrite_out, em.groups()[0]) # filtering syscalls here if needed. If it's not a match, # mark the errline so it is retried. if self.strip_syscalls != None and gotcall not in self.strip_syscalls: errline.skip = True return False - m = re.match(assignpat, runline) if m: if m.groups()[1] != gotcall: @@ -545,10 +550,12 @@ class Runner: with outfile, errfile: runlines = self.order_runfile(self.runfile) errline = errfile.readline() + errline = re.sub(pwrite_in, pwrite_out, errline) if re.match(dtruss_init_pat, errline): errline = errfile.readline() skiplines = False for runline in runlines: + runline = re.sub(pwrite_in, pwrite_out, runline) if runline == '...': skiplines = True if self.args.verbose: @@ -620,7 +627,8 @@ class Runner: elif self.args.systype == 'Darwin': # dtrace has no option to limit the syscalls to be traced, # so we'll filter the output. - self.strip_syscalls = self.headopts.trace_syscalls.split(',') + self.strip_syscalls = re.sub(pwrite_in, pwrite_out, + self.headopts.trace_syscalls).split(',') callargs.append(self.testexe) callargs.extend(self.runargs) diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run index 482eece476d..7d2c42ce64e 100644 --- a/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run @@ -46,15 +46,15 @@ SYSTEM("Darwin"); #define FTRUNCATE(fd, len) /* do nothing */ #endif -TRACE("close,fdatasync,fsync,ftruncate,open,pwrite,rename"); +TRACE("close,fdatasync,fsync,ftruncate,open,pwrite64,rename"); RUN(""); ... OUTPUT("--------------wiredtiger_open"); // lock == 3 lock = open("./WiredTiger.lock", O_RDWR|O_CREAT|O_CLOEXEC, 0666); -pwrite(lock, "WiredTiger lock file\n", 0x15, 0x0); +pwrite64(lock, "WiredTiger lock file\n", 0x15, 0x0); fd = open("./WiredTiger", O_RDWR|O_CREAT|O_CLOEXEC, 0666); -pwrite(fd, "WiredTiger\nWiredTiger"..., ...); +pwrite64(fd, "WiredTiger\nWiredTiger"..., ...); #ifdef __linux__ fdatasync(fd); #endif /* __linux__ */ @@ -63,7 +63,7 @@ close(fd); ... // On Linux, there are calls to open and read "/proc/meminfo" here. fd = open("./WiredTiger.basecfg.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); -pwrite(fd, "# Do not modify this file."..., ...); +pwrite64(fd, "# Do not modify this file."..., ...); #ifdef __linux__ fdatasync(fd); #endif /* __linux__ */ @@ -84,7 +84,7 @@ fdatasync(dir); close(dir); #endif /* __linux__ */ -pwrite(fd, ""..., 0x1000, 0x0); +pwrite64(fd, ""..., 0x1000, 0x0); #ifdef __linux__ fdatasync(fd); #endif /* __linux__ */ @@ -94,7 +94,7 @@ wt = OPEN_EXISTING("./WiredTiger.wt\0", O_RDWR|O_NOATIME|O_CLOEXEC); FTRUNCATE(wt, 0x1000); fd = open("./WiredTiger.turtle.set\0", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); -pwrite(fd, "WiredTiger version string\nWiredTiger"..., ...); +pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); #ifdef __linux__ fdatasync(fd); #endif /* __linux__ */ @@ -111,7 +111,7 @@ fdatasync(dir); close(dir); #endif /* __linux__ */ -pwrite(fd, ""..., 0x1000, 0x0); +pwrite64(fd, ""..., 0x1000, 0x0); #ifdef __linux__ fdatasync(fd); @@ -122,14 +122,17 @@ fd = OPEN_EXISTING("./WiredTigerLAS.wt", O_RDWR|O_NOATIME|O_CLOEXEC); FTRUNCATE(fd, 0x1000); fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); close(fd); -pwrite(wt, ""..., 0x1000, 0x1000); -pwrite(wt, ""..., 0x1000, 0x2000); -pwrite(wt, ""..., 0x1000, 0x3000); +pwrite64(wt, ""..., 0x1000, 0x1000); +pwrite64(wt, ""..., 0x1000, 0x2000); +pwrite64(wt, ""..., 0x1000, 0x3000); +#ifdef __linux__ +fdatasync(wt); +#endif /* __linux__ */ fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); close(fd); fd = open("./WiredTiger.turtle.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); -pwrite(fd, "WiredTiger version string\nWiredTiger"..., ...); +pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); #ifdef __linux__ fdatasync(fd); #endif /* __linux__ */ @@ -139,6 +142,7 @@ rename("./WiredTiger.turtle.set", "./WiredTiger.turtle"); dir = open("./", O_RDONLY); fdatasync(dir); close(dir); +fdatasync(wt); #endif /* __linux__ */ OUTPUT("--------------open_session"); @@ -150,7 +154,7 @@ dir = open("./", O_RDONLY); fdatasync(dir); close(dir); #endif /* __linux__ */ -pwrite(hello, "A\330\001"..., 0x1000, 0x0); +pwrite64(hello, "A\330\001"..., 0x1000, 0x0); #ifdef __linux__ fdatasync(hello); #endif /* __linux__ */ diff --git a/src/third_party/wiredtiger/tools/wtstats/stat_data.py b/src/third_party/wiredtiger/tools/wtstats/stat_data.py index 635e710c469..d925dd67b80 100644 --- a/src/third_party/wiredtiger/tools/wtstats/stat_data.py +++ b/src/third_party/wiredtiger/tools/wtstats/stat_data.py @@ -34,6 +34,9 @@ no_scale_per_second_list = [ 'reconciliation: split objects currently awaiting free', 'session: open cursor count', 'session: open session count', + 'session: table alter failed calls', + 'session: table alter successful calls', + 'session: table alter unchanged and skipped', 'session: table compact failed calls', 'session: table compact successful calls', 'session: table create failed calls', @@ -147,6 +150,9 @@ no_clear_list = [ 'reconciliation: split objects currently awaiting free', 'session: open cursor count', 'session: open session count', + 'session: table alter failed calls', + 'session: table alter successful calls', + 'session: table alter unchanged and skipped', 'session: table compact failed calls', 'session: table compact successful calls', 'session: table create failed calls', |