diff options
161 files changed, 3142 insertions, 1688 deletions
diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index d8c2de6e587..0dc38287155 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -170,7 +170,7 @@ config_free(CONFIG *cfg) if (config_opts[i].type == STRING_TYPE || config_opts[i].type == CONFIG_STRING_TYPE) { pstr = (char **) - ((unsigned char *)cfg + config_opts[i].offset); + ((u_char *)cfg + config_opts[i].offset); free(*pstr); *pstr = NULL; } @@ -431,7 +431,7 @@ config_opt(CONFIG *cfg, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) fprintf(stderr, "\t%s\n", config_opts[i].name); return (EINVAL); } - valueloc = ((unsigned char *)cfg + popt->offset); + valueloc = ((u_char *)cfg + popt->offset); switch (popt->type) { case BOOL_TYPE: if (v->type != WT_CONFIG_ITEM_BOOL) { @@ -560,7 +560,8 @@ config_opt_file(CONFIG *cfg, const char *filename) linenum++; /* Skip leading space. */ - for (ltrim = line; *ltrim && isspace(*ltrim); ltrim++) + for (ltrim = line; *ltrim && isspace((u_char)*ltrim); + ltrim++) ; /* @@ -579,7 +580,7 @@ config_opt_file(CONFIG *cfg, const char *filename) } /* Skip trailing space. */ - while (rtrim > ltrim && isspace(rtrim[-1])) + while (rtrim > ltrim && isspace((u_char)rtrim[-1])) rtrim--; /* @@ -599,7 +600,7 @@ config_opt_file(CONFIG *cfg, const char *filename) rtrim = comment; /* Skip trailing space again. */ - while (rtrim > ltrim && isspace(rtrim[-1])) + while (rtrim > ltrim && isspace((u_char)rtrim[-1])) rtrim--; /* diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 1a3d98d3e3e..9d35f6fa640 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -2608,7 +2608,7 @@ wtperf_rand(CONFIG_THREAD *thread) S2 = wtperf_value_range(cfg) * (cfg->pareto / 100.0) * (PARETO_SHAPE - 1); U = 1 - (double)rval / (double)UINT32_MAX; - rval = (pow(U, S1) - 1) * S2; + rval = (uint64_t)((pow(U, S1) - 1) * S2); /* * This Pareto calculation chooses out of range values about * 2% of the time, from my testing. That will lead to the diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 83fab4d6028..d874fa4eefe 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -320,6 +320,9 @@ extract_key(char *key_buf, uint64_t *keynop) * Print message and exit on failure. */ static inline void +die(int, const char *) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static inline void die(int e, const char *str) { fprintf(stderr, "Call to %s failed: %s", str, wiredtiger_strerror(e)); diff --git a/build_posix/aclocal/options.m4 b/build_posix/aclocal/options.m4 index 0fb49dbf1df..5f9b8748df2 100644 --- a/build_posix/aclocal/options.m4 +++ b/build_posix/aclocal/options.m4 @@ -215,6 +215,16 @@ pthread_adaptive|pthreads_adaptive) esac AC_MSG_RESULT($with_spinlock) +AC_MSG_CHECKING(if --enable-strict option specified) +AC_ARG_ENABLE(strict, + [AS_HELP_STRING([--enable-strict], + [Enable strict compiler checking.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_strict=no;; +*) wt_cv_enable_strict=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_strict) + AH_TEMPLATE(HAVE_VERBOSE, [Enable verbose message configuration.]) AC_MSG_CHECKING(if --enable-verbose option specified) AC_ARG_ENABLE(verbose, diff --git a/build_posix/aclocal/strict.m4 b/build_posix/aclocal/strict.m4 new file mode 100644 index 00000000000..b59f09fe584 --- /dev/null +++ b/build_posix/aclocal/strict.m4 @@ -0,0 +1,74 @@ +# AM_STRICT +# Per compiler-version flags used when compiling in strict mode. + +# GCC warnings. +AC_DEFUN([AM_GCC_WARNINGS], [ + w="$w -Wall -Wextra -Werror" + + w="$w -Waggregate-return" + w="$w -Wbad-function-cast" + w="$w -Wcast-align" + w="$w -Wdeclaration-after-statement" + w="$w -Wdouble-promotion" + w="$w -Wfloat-equal" + w="$w -Wformat-nonliteral" + w="$w -Wformat-security" + w="$w -Wformat=2" + w="$w -Winit-self" + w="$w -Wjump-misses-init" + w="$w -Wmissing-declarations" + w="$w -Wmissing-field-initializers" + w="$w -Wmissing-parameter-type" + w="$w -Wmissing-prototypes" + w="$w -Wnested-externs" + w="$w -Wold-style-definition" + w="$w -Wpacked" + w="$w -Wpointer-arith" + w="$w -Wpointer-sign" + w="$w -Wredundant-decls" + w="$w -Wshadow" + w="$w -Wsign-conversion" + w="$w -Wstrict-prototypes" + w="$w -Wswitch-enum" + w="$w -Wundef" + w="$w -Wunreachable-code" + w="$w -Wunsafe-loop-optimizations" + w="$w -Wunused" + w="$w -Wwrite-strings" + + # Non-fatal informational warnings. + w="$w -Wno-error=inline" + w="$w -Wno-error=unsafe-loop-optimizations" + + wt_cv_strict_warnings="$w" +]) + +# Clang warnings. +AC_DEFUN([AM_CLANG_WARNINGS], [ + w="-Weverything -Werror" + + w="$w -Wno-cast-align" + w="$w -Wno-documentation-unknown-command" + w="$w -Wno-format-nonliteral" + w="$w -Wno-packed" + w="$w -Wno-padded" + w="$w -Wno-reserved-id-macro" + w="$w -Wno-zero-length-array" + + # We should turn on cast-qual, but not as a fatal error: see WT-2690. + # For now, turn it off. + # w="$w -Wno-error=cast-qual" + w="$w -Wno-cast-qual" + + # Older OS X releases need some special love; these flags should be + # removed in the not-too-distant future. + # Apple clang version 4.1 + # (tags/Apple/clang-421.11.66) (based on LLVM 3.1svn) + w="$w -Wno-pedantic" + w="$w -Wno-unused-command-line-argument" + + # Ignore unrecognized options. + w="$w -Wno-unknown-warning-option" + + wt_cv_strict_warnings="$w" +]) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index 617304f9215..bbc6cf89d91 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -9,19 +9,20 @@ AC_CONFIG_AUX_DIR([build_posix/gnu-support]) AC_CONFIG_MACRO_DIR([build_posix/aclocal]) AC_CONFIG_SRCDIR([RELEASE_INFO]) -# If CFLAGS/CXXFLAGS were not set on entry, default to "-O3 -g" -: ${CFLAGS=-O3 -g} -: ${CXXFLAGS=-O3 -g} - # We rely on some automake features for testing (like AM_TEST_ENVIRONMENT) # that didn't work before 1.11.6. AM_INIT_AUTOMAKE([1.11.6 foreign parallel-tests subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([no])]) -# Configure options. The AM_OPTIONS and the libtool configuration -# need to stay here. Moving them below the compiler and other -# configurations causes -Wcast_align warnings and other warnings -# on OS X. +# If CFLAGS/CXXFLAGS were not set on entry, default to "-O3 -g" +: ${CFLAGS=-O3 -g} +: ${CXXFLAGS=-O3 -g} + +AC_PROG_CC(cc gcc) +AC_PROG_CXX(c++ g++) +AM_PROG_AS(as gas) + +# Configure options. AM_OPTIONS define([AC_LIBTOOL_LANG_CXX_CONFIG], [:])dnl @@ -30,9 +31,23 @@ LT_PREREQ(2.2.6) LT_INIT([pic-only]) AC_SUBST([LIBTOOL_DEPS]) -AC_PROG_CC(cc gcc) -AC_PROG_CXX(c++ g++) -AM_PROG_AS(as gas) +# If enable-strict is configured, turn on as much error checking as we can for +# this compiler. Intended for developers, and only works for gcc/clang, but it +# fills a need. +if test "$wt_cv_enable_strict" = "yes"; then + wt_cv_cc_version="`$CC --version | sed -eq`" + case "$wt_cv_cc_version" in + *clang*) + AM_CLANG_WARNINGS;; + *gcc*|*GCC*) + AM_GCC_WARNINGS;; + *) + AC_MSG_ERROR( + [--enable-strict does not support "$wt_cv_cc_version".]);; + esac + + AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings" +fi AM_CONDITIONAL([POSIX_HOST], [true]) AM_CONDITIONAL([WINDOWS_HOST], [false]) diff --git a/dist/api_data.py b/dist/api_data.py index f7b4cd6e03b..90b1c8378a2 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -439,7 +439,7 @@ connection_runtime_config = [ Config('file_max', '100MB', r''' the maximum size of log files''', min='100KB', max='2GB'), - Config('path', '', r''' + Config('path', '"."', r''' the path to a directory into which the log files are written. If the value is not an absolute path name, the files are created relative to the database home'''), @@ -959,6 +959,11 @@ methods = { Display the contents of on-disk blocks as they are verified, using the application's message handler, intended for debugging''', type='boolean'), + Config('dump_layout', 'false', r''' + Display the layout of the files as they are verified, using the + application's message handler, intended for debugging; requires + optional support from the block manager''', + type='boolean'), Config('dump_offsets', '', r''' Display the contents of specific on-disk blocks, using the application's message handler, intended for debugging''', @@ -967,10 +972,6 @@ methods = { Display the contents of in-memory pages as they are verified, using the application's message handler, intended for debugging''', type='boolean'), - Config('dump_shape', 'false', r''' - Display the shape of the tree after verification, - using the application's message handler, intended for debugging''', - type='boolean'), Config('strict', 'false', r''' Treat any verification problem as an error; by default, verify will warn, but not fail, in the case of errors that won't affect future diff --git a/dist/api_err.py b/dist/api_err.py index af5f1c12b37..82f961a4ac9 100644 --- a/dist/api_err.py +++ b/dist/api_err.py @@ -58,8 +58,6 @@ errors = [ more than the configured cache size to complete. The operation may be retried; if a transaction is in progress, it should be rolled back and the operation retried in a new transaction.'''), - Error('WT_PERM_DENIED', -31808, - 'permission denied (internal)', undoc=True), ] # Update the #defines in the wiredtiger.in file. diff --git a/dist/filelist b/dist/filelist index f3cb9514968..59624508cf0 100644 --- a/dist/filelist +++ b/dist/filelist @@ -107,6 +107,7 @@ src/meta/meta_turtle.c src/os_common/filename.c src/os_common/os_abort.c src/os_common/os_alloc.c +src/os_common/os_errno.c src/os_common/os_fhandle.c src/os_common/os_fs_inmemory.c src/os_common/os_fstream.c @@ -115,7 +116,6 @@ src/os_common/os_getopt.c src/os_common/os_strtouq.c src/os_posix/os_dir.c POSIX_HOST src/os_posix/os_dlopen.c POSIX_HOST -src/os_posix/os_errno.c POSIX_HOST src/os_posix/os_fallocate.c POSIX_HOST src/os_posix/os_fs.c POSIX_HOST src/os_posix/os_getenv.c POSIX_HOST @@ -132,7 +132,6 @@ src/os_posix/os_time.c POSIX_HOST src/os_posix/os_yield.c POSIX_HOST src/os_win/os_dir.c WINDOWS_HOST src/os_win/os_dlopen.c WINDOWS_HOST -src/os_win/os_errno.c WINDOWS_HOST src/os_win/os_fs.c WINDOWS_HOST src/os_win/os_getenv.c WINDOWS_HOST src/os_win/os_map.c WINDOWS_HOST @@ -147,6 +146,7 @@ src/os_win/os_snprintf.c WINDOWS_HOST src/os_win/os_thread.c WINDOWS_HOST src/os_win/os_time.c WINDOWS_HOST src/os_win/os_vsnprintf.c WINDOWS_HOST +src/os_win/os_winerr.c WINDOWS_HOST src/os_win/os_yield.c WINDOWS_HOST src/packing/pack_api.c src/packing/pack_impl.c diff --git a/dist/flags.py b/dist/flags.py index 48b0a2452a9..b5f36fb707a 100644 --- a/dist/flags.py +++ b/dist/flags.py @@ -50,6 +50,10 @@ flags = { 'TXN_LOG_CKPT_STOP', 'TXN_LOG_CKPT_SYNC', ], + 'txn_update_oldest' : [ + 'TXN_OLDEST_STRICT', + 'TXN_OLDEST_WAIT', + ], 'verbose' : [ 'VERB_API', 'VERB_BLOCK', @@ -94,6 +98,7 @@ flags = { 'CONN_LSM_MERGE', 'CONN_PANIC', 'CONN_READONLY', + 'CONN_RECOVERING', 'CONN_SERVER_ASYNC', 'CONN_SERVER_CHECKPOINT', 'CONN_SERVER_LSM', @@ -104,12 +109,12 @@ flags = { ], 'session' : [ 'SESSION_CAN_WAIT', - 'SESSION_CLEAR_EVICT_WALK', 'SESSION_INTERNAL', 'SESSION_LOCK_NO_WAIT', 'SESSION_LOCKED_CHECKPOINT', 'SESSION_LOCKED_HANDLE_LIST', 'SESSION_LOCKED_METADATA', + 'SESSION_LOCKED_PASS', 'SESSION_LOCKED_SCHEMA', 'SESSION_LOCKED_SLOT', 'SESSION_LOCKED_TABLE', diff --git a/dist/log.py b/dist/log.py index 9201b20054b..8743dd3a71c 100644 --- a/dist/log.py +++ b/dist/log.py @@ -178,7 +178,7 @@ __wt_logop_read(WT_SESSION_IMPL *session, } static size_t -__logrec_json_unpack_str(char *dest, size_t destlen, const char *src, +__logrec_json_unpack_str(char *dest, size_t destlen, const u_char *src, size_t srclen) { \tsize_t total; diff --git a/dist/s_copyright.list b/dist/s_copyright.list index c6a5910087b..4999d2a37a2 100644 --- a/dist/s_copyright.list +++ b/dist/s_copyright.list @@ -29,6 +29,8 @@ skip src/config/config_def.c skip src/conn/api_strerror.c skip src/docs/tools/doxypy.py skip src/include/extern.h +skip src/include/extern_posix.h +skip src/include/extern_win.h skip src/include/flags.h skip src/include/queue.h skip src/log/log_auto.c diff --git a/dist/s_define.list b/dist/s_define.list index c9777c86675..2cdda74e7d4 100644 --- a/dist/s_define.list +++ b/dist/s_define.list @@ -23,6 +23,7 @@ WT_CONN_CHECK_PANIC WT_DEADLOCK WT_DEBUG_BYTE WT_ERR_ERROR_OK +WT_EXT_FOREACH_OFF WT_HANDLE_CLOSED WT_HANDLE_NULLABLE WT_LOG_SLOT_ACTIVE @@ -42,13 +43,17 @@ WT_STATS_FIELD_TO_SLOT WT_STATS_SLOT_ID WT_STAT_DECR WT_STAT_DECRV +WT_STAT_DECRV_ATOMIC WT_STAT_FAST_CONN_DECRV WT_STAT_FAST_DATA_DECRV WT_STAT_FAST_DECR WT_STAT_FAST_DECRV +WT_STAT_FAST_DECRV_ATOMIC WT_STAT_FAST_INCR WT_STAT_FAST_INCRV +WT_STAT_FAST_INCRV_ATOMIC WT_STAT_FAST_SET +WT_STAT_INCRV_ATOMIC WT_STAT_WRITE WT_TIMEDIFF_US WT_TRET_ERROR_OK diff --git a/dist/s_label b/dist/s_label index b7c5795234a..b56ecc6fc78 100755 --- a/dist/s_label +++ b/dist/s_label @@ -23,7 +23,7 @@ file_parse() # where there's a jump to the error label after the error label. for f in `find bench examples ext src test -name '*.[ci]'`; do file_parse $f | - egrep '(WT_ERR|WT_ILLEGAL_VALUE_ERR)\(.*(WT_ILLEGAL_VALUE|WT_RET)\(.*err:|[^a-z_]err:.*(WT_ERR|WT_ILLEGAL_VALUE_ERR)\(' | + egrep '(WT_ERR[_A-Z]*|WT_ILLEGAL_VALUE_ERR)\(.*(WT_ILLEGAL_VALUE|WT_RET[_A-Z]*)\(.*err:|[^a-z_]err:.*(WT_ERR|WT_ILLEGAL_VALUE_ERR)\(' | sed 's/:.*//' > $t test -s $t && { @@ -32,6 +32,14 @@ for f in `find bench examples ext src test -name '*.[ci]'`; do } done +# Returns before jumps to an error label within the same loop. +# Jumps before returns have already been detected above. +for f in `find bench examples ext src test -name '*.[ci]'`; do + file_parse $f | sed "s=^=$f:=" +done | python dist/s_label_loop.py | + egrep '\{@[^@]*(WT_ILLEGAL_VALUE|WT_RET[_A-Z]*)\([^@]*(WT_ERR[_A-Z]*|WT_ILLEGAL_VALUE_ERR)\(.*err:' | + sed -e 's/^\([^:]*\): *\([^:]*\):.*/\1:\2: mix of returns and jump to the error label within a loop/' + # Return of 0 in functions after a jump to the error label. for f in `find bench examples ext src test -name '*.[ci]'`; do file_parse $f | diff --git a/dist/s_label_loop.py b/dist/s_label_loop.py new file mode 100644 index 00000000000..5cc222a4250 --- /dev/null +++ b/dist/s_label_loop.py @@ -0,0 +1,28 @@ +# Mark outer loop boundaries with {@ and }@ . Nested loops are not marked. +# Each input line is the content of a C function. +import re, sys + +p = re.compile('((for |while |_FOREACH|FOREACH_BEGIN)\([^{)]*\)|do) {') +for line in sys.stdin: + matched = 0 + m = p.search(line) + while m != None: + matched = 1 + pos = m.end() + out = line[:pos] + "@" + level = 1 + length = len(line) + while level > 0 and pos < length: + c = line[pos:pos+1] + pos += 1 + out += c + if c == "}": + level -= 1 + elif c == "{": + level += 1 + out += "@" + sys.stdout.write(out) + line = line[pos:] + m = p.search(line) + if matched != 0: + sys.stdout.write(line) diff --git a/dist/s_longlines b/dist/s_longlines index 2837d400f9e..91dada361f4 100755 --- a/dist/s_longlines +++ b/dist/s_longlines @@ -11,6 +11,8 @@ l=`(cd .. && sed -e '/checksum\/power8/d' \ -e '/dist\/stat_data\.py/d' \ -e '/include\/extern\.h/d' \ + -e '/include\/extern_posix\.h/d' \ + -e '/include\/extern_win\.h/d' \ -e '/support\/stat\.c/d'` for f in $l ; do diff --git a/dist/s_prototypes b/dist/s_prototypes index 01d1e9bb4c3..73f7be371ea 100755 --- a/dist/s_prototypes +++ b/dist/s_prototypes @@ -28,36 +28,52 @@ proto() -e 's/\* /\*/g' \ -e 's/ */ /g' \ -e 's/^/extern /' \ - -e 's/WT_GCC_FUNC_/WT_GCC_FUNC_DECL_/' \ - -e 's/$/;/p' < $1 + -e 's/WT_GCC_FUNC_/WT_GCC_FUNC_DECL_/g' \ + -e '# If a line ends in #endif, appending a semicolon will result' \ + -e '# in an illegal expression, force an appended newline using' \ + -e '# the H command because substitute may not allow newline in' \ + -e '# the RHS of the expression.' \ + -e '/#endif$/{' \ + -e x \ + -e 's/.*//' \ + -e H \ + -e x \ + -e '}' \ + -e 's/$/;/' \ + -e p < $1 } -( -cat <<EOF +# proto -- +# generate the list of prototypes given a file list +externs() +{ +(cat <<EOF /* DO NOT EDIT: automatically built by dist/s_prototypes. */ EOF + for i in $l; do + proto ../$i + done) > $t + cmp $t $f > /dev/null 2>&1 || + (echo "Building $f" && rm -f $f && cp $t $f) +} -# First, get prototypes for everything but the OS directories. -# Second, get prototypes for the OS directories. -# The reason for this is because the OS directories repeat names (that is, there -# are common names in both os_posix and os_win), and so we sort the prototypes -# to avoid repeating them in the output (which some compilers won't tolerate). -# We'd sort everything and discard duplicates, but we can't sort when function -# signatures are on multiple lines, that is, #ifdef'd function signatures. Since -# the OS directories are the only places with repeated names, and they have no -# #ifdef'd signatures, we do it this way. -l=`sed -e '/^[a-z]/!d' -e '/src\/os/d' -e 's/[ ].*$//' filelist` -for i in $l; do - proto ../$i -done -l=`echo ../src\/os*/*.c` +f=../src/include/extern_win.h +l=`sed \ + -e '/os_win/!d' \ + -e 's/[ ].*$//' filelist` +externs -for i in $l; do - proto $i -done | env LC_ALL=C sort -u -) > $t +f=../src/include/extern_posix.h +l=`sed \ + -e '/os_posix/!d' \ + -e 's/[ ].*$//' filelist` +externs f=../src/include/extern.h -cmp $t $f > /dev/null 2>&1 || - (echo "Building $f" && rm -f $f && cp $t $f) +l=`sed \ + -e '/^[a-z]/!d' \ + -e '/os_posix/d' \ + -e '/os_win/d' \ + -e 's/[ ].*$//' filelist` +externs diff --git a/dist/s_string.ok b/dist/s_string.ok index d45cace728a..7966ff2cf2e 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -87,6 +87,7 @@ Decrement Decrypt DeleteFileA EAGAIN +EB EBUSY EEXIST EINTR @@ -117,6 +118,7 @@ FNV FORALL FOREACH FULLFSYNC +FindClose FindFirstFile Fixup Fk @@ -131,6 +133,7 @@ GIDs Gcc Geoff GetEnvironmentVariableA +GetFileAttributesA GetFileAttributesEx GetFileSizeEx GetLastError @@ -213,6 +216,7 @@ Mewhort Mitzenmacher MongoDB MoveFile +MoveFileA Multi Multithreaded Mutex @@ -471,6 +475,7 @@ ckptfrag ckptlist cksum cloexec +closedir clsm cmd cmp @@ -644,6 +649,7 @@ fmterr fnv foc fopen +formatmessage fp fprintf free'd @@ -695,6 +701,7 @@ ibackup icount idx ifdef's +iiu ikey im impl @@ -728,9 +735,20 @@ intrin inuse io ip +isalnum +isalpha +iscntrl +isdigit +isgraph islocked +islower ispo +isprint +ispunct isrc +isspace +isupper +isxdigit iter iteratively jnr @@ -1057,7 +1075,9 @@ toklen tokname tokstart toktype +tolower totalsec +toupper transactional transactionally trecno @@ -1088,6 +1108,7 @@ unescaped unicode uninstantiated unistd +unlink unlinked unmap unmarshall diff --git a/dist/s_style b/dist/s_style index 85220124971..a222c004cc3 100755 --- a/dist/s_style +++ b/dist/s_style @@ -60,11 +60,13 @@ else echo "$f: use TAILQ for all lists" fi - if ! expr "$f" : 'src/os_common/.*' > /dev/null && + if ! expr "$f" : 'src/include/extern.h' > /dev/null && + ! expr "$f" : 'src/include/extern_posix.h' > /dev/null && + ! expr "$f" : 'src/include/extern_win.h' > /dev/null && + ! expr "$f" : 'src/include/os.h' > /dev/null && + ! expr "$f" : 'src/os_common/.*' > /dev/null && ! expr "$f" : 'src/os_posix/.*' > /dev/null && ! expr "$f" : 'src/os_win/.*' > /dev/null && - ! expr "$f" : 'src/include/extern.h' > /dev/null && - ! expr "$f" : 'src/include/os.h' > /dev/null && grep '__wt_errno' $f > $t; then echo "$f: upper-level code should not call __wt_errno" cat $t @@ -73,7 +75,7 @@ else if ! expr "$f" : 'examples/c/.*' > /dev/null && ! expr "$f" : 'ext/datasources/helium/helium.c' > /dev/null && ! expr "$f" : 'src/include/os.h' > /dev/null && - grep "%zu" $f | grep -v 'SIZET_FMT' > $t; then + egrep "%[0-9]*zu" $f | grep -v 'SIZET_FMT' > $t; then echo "$f: %zu needs to be fixed for Windows" cat $t fi @@ -138,6 +140,20 @@ else } fi + # Use of ctype functions that sign extend their arguments. + if ! expr "$f" : 'bench/.*' > /dev/null && + ! expr "$f" : 'test/csuite/.*' > /dev/null && + ! expr "$f" : 'examples/.*' > /dev/null && + ! expr "$f" : 'ext/.*' > /dev/null && + ! expr "$f" : 'src/include/ctype.i' > /dev/null; then + if egrep '(#include.*["</]ctype.h[">]|\b(is(alnum|alpha|cntrl|digit|graph|lower|print|punct|space|upper|xdigit)|to(lower|toupper))\()' $f > $t; then + test -s $t && { + echo "$f: direct use of ctype.h functions, instead of ctype.i equivalents" + cat $t + } + fi + fi + tr -cd '[:alnum:][:space:][:punct:]' < $f | unexpand | sed -e 's/){/) {/' \ diff --git a/dist/stat_data.py b/dist/stat_data.py index 48066c11700..694ffc86ee4 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -81,6 +81,10 @@ class SessionStat(Stat): prefix = 'session' def __init__(self, name, desc, flags=''): Stat.__init__(self, name, SessionStat.prefix, desc, flags) +class ThreadState(Stat): + prefix = 'thread-state' + def __init__(self, name, desc, flags=''): + Stat.__init__(self, name, ThreadState.prefix, desc, flags) class TxnStat(Stat): prefix = 'transaction' def __init__(self, name, desc, flags=''): @@ -97,10 +101,20 @@ class YieldStat(Stat): ########################################## groups = {} groups['cursor'] = [CursorStat.prefix, SessionStat.prefix] -groups['evict'] = [CacheStat.prefix, ConnStat.prefix, BlockStat.prefix] +groups['evict'] = [ + BlockStat.prefix, + CacheStat.prefix, + ConnStat.prefix, + ThreadState.prefix +] groups['lsm'] = [LSMStat.prefix, TxnStat.prefix] groups['memory'] = [CacheStat.prefix, ConnStat.prefix, RecStat.prefix] -groups['system'] = [ConnStat.prefix, DhandleStat.prefix, SessionStat.prefix] +groups['system'] = [ + ConnStat.prefix, + DhandleStat.prefix, + SessionStat.prefix, + ThreadState.prefix +] ########################################## # CONNECTION statistics @@ -113,6 +127,7 @@ connection_stats = [ ConnStat('cond_auto_wait_reset', 'auto adjusting condition resets'), ConnStat('cond_wait', 'pthread mutex condition wait calls'), ConnStat('file_open', 'files currently open', 'no_clear,no_scale'), + ConnStat('fsync_io', 'total fsync I/Os'), ConnStat('memory_allocation', 'memory allocations'), ConnStat('memory_free', 'memory frees'), ConnStat('memory_grow', 'memory re-allocations'), @@ -162,6 +177,7 @@ connection_stats = [ CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), CacheStat('cache_eviction_aggressive_set', 'eviction currently operating in aggressive mode', 'no_clear,no_scale'), CacheStat('cache_eviction_app', 'pages evicted by application threads'), + CacheStat('cache_eviction_app_dirty', 'modified pages evicted by application threads'), CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'), CacheStat('cache_eviction_clean', 'unmodified pages evicted'), CacheStat('cache_eviction_deepen', 'page split during eviction deepened the tree'), @@ -176,6 +192,9 @@ connection_stats = [ CacheStat('cache_eviction_hazard', 'hazard pointer blocked page eviction'), CacheStat('cache_eviction_internal', 'internal pages evicted'), CacheStat('cache_eviction_maximum_page_size', 'maximum page size at eviction', 'no_clear,no_scale,size'), + CacheStat('cache_eviction_pages_queued', 'pages queued for eviction'), + CacheStat('cache_eviction_pages_queued_oldest', 'pages queued for urgent eviction'), + CacheStat('cache_eviction_pages_seen', 'pages seen by eviction walk'), CacheStat('cache_eviction_queue_empty', 'eviction server candidate queue empty when topping up'), CacheStat('cache_eviction_queue_not_empty', 'eviction server candidate queue not empty when topping up'), CacheStat('cache_eviction_server_evicting', 'eviction server evicting pages'), @@ -186,6 +205,8 @@ connection_stats = [ CacheStat('cache_eviction_split_internal', 'internal pages split during eviction'), CacheStat('cache_eviction_split_leaf', 'leaf pages split during eviction'), CacheStat('cache_eviction_walk', 'pages walked for eviction'), + CacheStat('cache_eviction_walks_active', 'files with active eviction walks', 'no_clear,no_scale,size'), + CacheStat('cache_eviction_walks_started', 'files with new eviction walks started'), CacheStat('cache_eviction_worker_evicting', 'eviction worker thread evicting pages'), CacheStat('cache_hazard_checks', 'hazard pointer check calls'), CacheStat('cache_hazard_max', 'hazard pointer maximum array length', 'max_aggregate,no_scale'), @@ -250,6 +271,8 @@ connection_stats = [ LogStat('log_slot_unbuffered', 'consolidated slot unbuffered writes'), LogStat('log_sync', 'log sync operations'), LogStat('log_sync_dir', 'log sync_dir operations'), + LogStat('log_sync_dir_duration', 'log sync_dir time duration (usecs)'), + LogStat('log_sync_duration', 'log sync time duration (usecs)'), LogStat('log_write_lsn', 'log server thread advances write LSN'), LogStat('log_write_lsn_skip', 'log server thread write LSN walk skipped'), LogStat('log_writes', 'log write operations'), @@ -270,6 +293,10 @@ connection_stats = [ ########################################## TxnStat('txn_begin', 'transaction begins'), TxnStat('txn_checkpoint', 'transaction checkpoints'), + TxnStat('txn_checkpoint_fsync_post', 'transaction fsync calls for checkpoint after allocating the transaction ID'), + TxnStat('txn_checkpoint_fsync_post_duration', 'transaction fsync duration for checkpoint after allocating the transaction ID (usecs)'), + TxnStat('txn_checkpoint_fsync_pre', 'transaction fsync calls for checkpoint before allocating the transaction ID'), + TxnStat('txn_checkpoint_fsync_pre_duration', 'transaction fsync duration for checkpoint before allocating the transaction ID (usecs)'), TxnStat('txn_checkpoint_generation', 'transaction checkpoint generation', 'no_clear,no_scale'), TxnStat('txn_checkpoint_running', 'transaction checkpoint currently running', 'no_clear,no_scale'), TxnStat('txn_checkpoint_time_max', 'transaction checkpoint max time (msecs)', 'no_clear,no_scale'), @@ -322,6 +349,13 @@ connection_stats = [ CursorStat('cursor_update', 'cursor update calls'), ########################################## + # Thread State statistics + ########################################## + ThreadState('fsync_active', 'active filesystem fsync calls','no_clear,no_scale'), + ThreadState('read_active', 'active filesystem read calls','no_clear,no_scale'), + ThreadState('write_active', 'active filesystem write calls','no_clear,no_scale'), + + ########################################## # Yield statistics ########################################## YieldStat('page_busy_blocked', 'page acquire busy blocked'), @@ -480,9 +514,11 @@ dsrc_stats = sorted(dsrc_stats, key=attrgetter('desc')) # Cursor Join statistics ########################################## join_stats = [ - JoinStat('accesses', 'accesses'), - JoinStat('actual_count', 'actual count of items'), JoinStat('bloom_false_positive', 'bloom filter false positives'), + JoinStat('bloom_insert', 'items inserted into a bloom filter'), + JoinStat('iterated', 'items iterated'), + JoinStat('main_access', 'accesses to the main table'), + JoinStat('membership_check', 'checks that conditions of membership are satisfied'), ] join_stats = sorted(join_stats, key=attrgetter('desc')) diff --git a/examples/c/ex_data_source.c b/examples/c/ex_data_source.c index c550b337b78..6ed80dfcf19 100644 --- a/examples/c/ex_data_source.c +++ b/examples/c/ex_data_source.c @@ -58,6 +58,17 @@ my_create(WT_DATA_SOURCE *dsrc, WT_SESSION *session, (void)config; { +#if !defined(ERROR_BAD_COMMAND) +#define ERROR_BAD_COMMAND 37 +#endif + /*! [WT_EXTENSION_API map_windows_error] */ + int posix_error = + wt_api->map_windows_error(wt_api, session, ERROR_BAD_COMMAND); + /*! [WT_EXTENSION_API map_windows_error] */ + (void)posix_error; + } + + { const char *msg = "string"; /*! [WT_EXTENSION_API err_printf] */ (void)wt_api->err_printf( diff --git a/examples/c/ex_encrypt.c b/examples/c/ex_encrypt.c index a919148aff0..3b3323bc091 100644 --- a/examples/c/ex_encrypt.c +++ b/examples/c/ex_encrypt.c @@ -122,8 +122,8 @@ do_rotate(char *buf, size_t len, int rotn) * Now rotate */ for (i = 0; i < len; i++) - if (isalpha(buf[i])) { - if (islower(buf[i])) + if (isalpha((unsigned char)buf[i])) { + if (islower((unsigned char)buf[i])) buf[i] = ((buf[i] - 'a') + rotn) % 26 + 'a'; else buf[i] = ((buf[i] - 'A') + rotn) % 26 + 'A'; diff --git a/examples/c/ex_event_handler.c b/examples/c/ex_event_handler.c index f2cdd00d9d7..7122e71882e 100644 --- a/examples/c/ex_event_handler.c +++ b/examples/c/ex_event_handler.c @@ -68,7 +68,7 @@ handle_wiredtiger_error(WT_EVENT_HANDLER *handler, /* Report the error on the console. */ fprintf(stderr, "app_id %s, thread context %p, error %d, message %s\n", - custom_handler->app_id, session, error, message); + custom_handler->app_id, (void *)session, error, message); return (0); } @@ -83,7 +83,8 @@ handle_wiredtiger_message( { /* Cast the handler back to our custom handler. */ printf("app id %s, thread context %p, message %s\n", - ((CUSTOM_EVENT_HANDLER *)handler)->app_id, session, message); + ((CUSTOM_EVENT_HANDLER *)handler)->app_id, + (void *)session, message); return (0); } diff --git a/examples/c/ex_file_system.c b/examples/c/ex_file_system.c index 6ba0b95c338..77e8f40480b 100644 --- a/examples/c/ex_file_system.c +++ b/examples/c/ex_file_system.c @@ -28,35 +28,57 @@ * ex_file_system.c * demonstrates how to use the custom file system interface */ -#include <errno.h> -#include <inttypes.h> -#include <stdlib.h> -#include <string.h> -#if (defined(_WIN32) && _MSC_VER < 1900) -/* snprintf is not supported on <= VS2013 */ -#define snprintf _snprintf -#endif - -#include <wiredtiger.h> -#include <wiredtiger_ext.h> - -#include "queue_example.h" -static const char *home; +/* + * Include WiredTiger internal functions: we need architecture portable locking + * in this example, and we use the TAILQ_XXX functions to keep the code simple. + * + * Application-writers SHOULD NOT INCLUDE "wt_internal.h", the public WiredTiger + * include files should be used instead: + * + * #include <wiredtiger.h> + * #include <wiredtiger_ext.h> + */ +#include "wt_internal.h" /* - * Example file system implementation. Using memory buffers to represent files. + * This example code uses internal WiredTiger functions for portable locking. + * We use #defines to clarify the meaning and ignore errors to simplify the + * code. * - * WARNING: This implementation isn't thread safe: WiredTiger performs schema - * and I/O operations in parallel, so all access to the handle must be thread- - * safe. + * Application writers SHOULD NOT COPY THIS LOCKING CODE, it's special-case code + * to make this example portable across platforms. + */ +#define ALLOCATE_FILE_SYSTEM_LOCK(demo_fs) \ + (void)__wt_spin_init(NULL, &(demo_fs)->lock, "demo file handle lock") +#define DESTROY_FILE_SYSTEM_LOCK(wt_session, demo_fs) \ + __wt_spin_destroy((WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) +#define LOCK_FILE_SYSTEM(wt_session, demo_fs) \ + __wt_spin_lock((WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) +#define UNLOCK_FILE_SYSTEM(wt_session, demo_fs) \ + __wt_spin_unlock( \ + (WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) + +/* + * Example file system implementation, using memory buffers to represent files. */ typedef struct { WT_FILE_SYSTEM iface; + /* + * WiredTiger performs schema and I/O operations in parallel, all file + * system and file handle access must be thread-safe. This example uses + * a single, global file system lock for simplicity; real applications + * might require finer granularity, for example, a single lock for the + * file system handle list and per-handle locks serializing I/O. + */ + WT_SPINLOCK lock; /* Lock */ + int opened_file_count; int opened_unique_file_count; int closed_file_count; + int read_ops; + int write_ops; /* Queue of file handles */ TAILQ_HEAD(demo_file_handle_qh, demo_file_handle) fileq; @@ -71,15 +93,14 @@ typedef struct demo_file_handle { /* * Add custom file handle fields after the interface. */ - DEMO_FILE_SYSTEM *demo_fs; + DEMO_FILE_SYSTEM *demo_fs; /* Enclosing file system */ - TAILQ_ENTRY(demo_file_handle) q; + TAILQ_ENTRY(demo_file_handle) q; /* Queue of handles */ uint32_t ref; /* Reference count */ char *buf; /* In-memory contents */ size_t bufsize; /* In-memory buffer size */ - size_t off; /* Read/write offset */ size_t size; /* Read/write data size */ } DEMO_FILE_HANDLE; @@ -122,7 +143,6 @@ static int demo_file_read( WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, void *); static int demo_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); static int demo_file_sync(WT_FILE_HANDLE *, WT_SESSION *); -static int demo_file_sync_nowait(WT_FILE_HANDLE *, WT_SESSION *); static int demo_file_truncate(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t); static int demo_file_write( WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, const void *); @@ -170,7 +190,14 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) demo_fs->wtext = wtext; file_system = (WT_FILE_SYSTEM *)demo_fs; - /* Retrieve our configuration information, the "config" value. */ + /* + * Applications may have their own configuration information to pass to + * the underlying filesystem implementation. See the main function for + * the setup of those configuration strings; here we parse configuration + * information as passed in by main, through WiredTiger. + * + * Retrieve our configuration information, the "config" value. + */ if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) { (void)wtext->err_printf(wtext, NULL, "WT_EXTENSION_API.config_get: config: %s", @@ -223,6 +250,8 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) goto err; } + ALLOCATE_FILE_SYSTEM_LOCK(demo_fs); + /* Initialize the in-memory jump table. */ file_system->fs_directory_list = demo_fs_directory_list; file_system->fs_directory_list_free = demo_fs_directory_list_free; @@ -260,51 +289,63 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, DEMO_FILE_SYSTEM *demo_fs; WT_EXTENSION_API *wtext; WT_FILE_HANDLE *file_handle; + int ret = 0; (void)file_type; /* Unused */ (void)flags; /* Unused */ + *file_handlep = NULL; + demo_fs = (DEMO_FILE_SYSTEM *)file_system; demo_fh = NULL; wtext = demo_fs->wtext; + LOCK_FILE_SYSTEM(session, demo_fs); ++demo_fs->opened_file_count; /* * First search the file queue, if we find it, assert there's only a - * single reference, we only supports a single handle on any file. + * single reference, we only support a single handle on any file. */ demo_fh = demo_handle_search(file_system, name); if (demo_fh != NULL) { if (demo_fh->ref != 0) { (void)wtext->err_printf(wtext, session, "demo_fs_open: %s: file already open", name); - return (EBUSY); + ret = EBUSY; + goto err; } demo_fh->ref = 1; - demo_fh->off = 0; *file_handlep = (WT_FILE_HANDLE *)demo_fh; + + UNLOCK_FILE_SYSTEM(session, demo_fs); return (0); } /* The file hasn't been opened before, create a new one. */ - if ((demo_fh = calloc(1, sizeof(DEMO_FILE_HANDLE))) == NULL) - return (ENOMEM); + if ((demo_fh = calloc(1, sizeof(DEMO_FILE_HANDLE))) == NULL) { + ret = ENOMEM; + goto err; + } /* Initialize private information. */ - demo_fh->ref = 1; - demo_fh->off = demo_fh->size = 0; demo_fh->demo_fs = demo_fs; - if ((demo_fh->buf = calloc(1, DEMO_FILE_SIZE_INCREMENT)) == NULL) - goto enomem; + demo_fh->ref = 1; + if ((demo_fh->buf = calloc(1, DEMO_FILE_SIZE_INCREMENT)) == NULL) { + ret = ENOMEM; + goto err; + } demo_fh->bufsize = DEMO_FILE_SIZE_INCREMENT; + demo_fh->size = 0; /* Initialize public information. */ file_handle = (WT_FILE_HANDLE *)demo_fh; - if ((file_handle->name = strdup(name)) == NULL) - goto enomem; + if ((file_handle->name = strdup(name)) == NULL) { + ret = ENOMEM; + goto err; + } /* * Setup the function call table for our custom file system. Set the @@ -323,7 +364,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, file_handle->fh_read = demo_file_read; file_handle->fh_size = demo_file_size; file_handle->fh_sync = demo_file_sync; - file_handle->fh_sync_nowait = demo_file_sync_nowait; + file_handle->fh_sync_nowait = NULL; file_handle->fh_truncate = demo_file_truncate; file_handle->fh_write = demo_file_write; @@ -331,11 +372,14 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, ++demo_fs->opened_unique_file_count; *file_handlep = file_handle; - return (0); -enomem: free(demo_fh->buf); - free(demo_fh); - return (ENOMEM); + if (0) { +err: free(demo_fh->buf); + free(demo_fh); + } + + UNLOCK_FILE_SYSTEM(session, demo_fs); + return (ret); } /* @@ -349,21 +393,25 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, { DEMO_FILE_HANDLE *demo_fh; DEMO_FILE_SYSTEM *demo_fs; - size_t dir_len, prefix_len; - char *name, **entries; + size_t len, prefix_len; uint32_t allocated, count; - - (void)session; /* Unused */ + int ret = 0; + char *name, **entries; demo_fs = (DEMO_FILE_SYSTEM *)file_system; + + *dirlistp = NULL; + *countp = 0; + entries = NULL; allocated = count = 0; - dir_len = strlen(directory); + len = strlen(directory); prefix_len = prefix == NULL ? 0 : strlen(prefix); + LOCK_FILE_SYSTEM(session, demo_fs); TAILQ_FOREACH(demo_fh, &demo_fs->fileq, q) { name = demo_fh->iface.name; - if (strncmp(name, directory, dir_len) != 0 || + if (strncmp(name, directory, len) != 0 || (prefix != NULL && strncmp(name, prefix, prefix_len) != 0)) continue; @@ -374,8 +422,10 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, if (count >= allocated) { entries = realloc( entries, (allocated + 10) * sizeof(char *)); - if (entries == NULL) - return (ENOMEM); + if (entries == NULL) { + ret = ENOMEM; + goto err; + } memset(entries + allocated * sizeof(char *), 0, 10 * sizeof(char *)); allocated += 10; @@ -386,7 +436,17 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, *dirlistp = entries; *countp = count; - return (0); +err: UNLOCK_FILE_SYSTEM(session, demo_fs); + if (ret == 0) + return (0); + + if (entries != NULL) { + while (count > 0) + free(entries[--count]); + free(entries); + } + + return (ret); } /* @@ -431,10 +491,13 @@ static int demo_fs_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, bool *existp) { - (void)session; /* Unused */ + DEMO_FILE_SYSTEM *demo_fs; - *existp = - demo_handle_search(file_system, name) != NULL; + demo_fs = (DEMO_FILE_SYSTEM *)file_system; + + LOCK_FILE_SYSTEM(session, demo_fs); + *existp = demo_handle_search(file_system, name) != NULL; + UNLOCK_FILE_SYSTEM(session, demo_fs); return (0); } @@ -447,12 +510,17 @@ static int demo_fs_remove( WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name) { + DEMO_FILE_SYSTEM *demo_fs; DEMO_FILE_HANDLE *demo_fh; - int ret; + int ret = 0; + + demo_fs = (DEMO_FILE_SYSTEM *)file_system; ret = ENOENT; + LOCK_FILE_SYSTEM(session, demo_fs); if ((demo_fh = demo_handle_search(file_system, name)) != NULL) ret = demo_handle_remove(session, demo_fh); + UNLOCK_FILE_SYSTEM(session, demo_fs); return (ret); } @@ -466,19 +534,23 @@ demo_fs_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *from, const char *to) { DEMO_FILE_HANDLE *demo_fh; + DEMO_FILE_SYSTEM *demo_fs; char *copy; + int ret = 0; - (void)session; /* Unused */ + demo_fs = (DEMO_FILE_SYSTEM *)file_system; + LOCK_FILE_SYSTEM(session, demo_fs); if ((demo_fh = demo_handle_search(file_system, from)) == NULL) - return (ENOENT); - - if ((copy = strdup(to)) == NULL) - return (ENOMEM); - - free(demo_fh->iface.name); - demo_fh->iface.name = copy; - return (0); + ret = ENOENT; + else if ((copy = strdup(to)) == NULL) + ret = ENOMEM; + else { + free(demo_fh->iface.name); + demo_fh->iface.name = copy; + } + UNLOCK_FILE_SYSTEM(session, demo_fs); + return (ret); } /* @@ -489,13 +561,17 @@ static int demo_fs_size(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, wt_off_t *sizep) { + DEMO_FILE_SYSTEM *demo_fs; DEMO_FILE_HANDLE *demo_fh; int ret = 0; + demo_fs = (DEMO_FILE_SYSTEM *)file_system; + ret = ENOENT; + LOCK_FILE_SYSTEM(session, demo_fs); if ((demo_fh = demo_handle_search(file_system, name)) != NULL) - ret = demo_file_size( - (WT_FILE_HANDLE *)demo_fh, session, sizep); + ret = demo_file_size((WT_FILE_HANDLE *)demo_fh, session, sizep); + UNLOCK_FILE_SYSTEM(session, demo_fs); return (ret); } @@ -511,8 +587,6 @@ demo_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) DEMO_FILE_SYSTEM *demo_fs; int ret = 0, tret; - (void)session; /* Unused */ - demo_fs = (DEMO_FILE_SYSTEM *)file_system; while ((demo_fh = TAILQ_FIRST(&demo_fs->fileq)) != NULL) @@ -522,8 +596,12 @@ demo_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) printf("Custom file system\n"); printf("\t%d unique file opens\n", demo_fs->opened_unique_file_count); - printf("\t%d opened\n", demo_fs->opened_file_count); - printf("\t%d closed\n", demo_fs->closed_file_count); + printf("\t%d files opened\n", demo_fs->opened_file_count); + printf("\t%d files closed\n", demo_fs->closed_file_count); + printf("\t%d reads, %d writes\n", + demo_fs->read_ops, demo_fs->write_ops); + + DESTROY_FILE_SYSTEM_LOCK(session, demo_fs); free(demo_fs); return (ret); @@ -537,21 +615,15 @@ static int demo_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) { DEMO_FILE_HANDLE *demo_fh; - WT_EXTENSION_API *wtext; + DEMO_FILE_SYSTEM *demo_fs; demo_fh = (DEMO_FILE_HANDLE *)file_handle; - wtext = demo_fh->demo_fs->wtext; - - if (demo_fh->ref < 1) { - (void)wtext->err_printf(wtext, session, - "demo_file_close: %s: handle already closed", - demo_fh->iface.name); - return (EINVAL); - } - --demo_fh->ref; + demo_fs = demo_fh->demo_fs; - if (demo_fh->ref == 0) - ++demo_fh->demo_fs->closed_file_count; + LOCK_FILE_SYSTEM(session, demo_fs); + if (--demo_fh->ref == 0) + ++demo_fs->closed_file_count; + UNLOCK_FILE_SYSTEM(session, demo_fs); return (0); } @@ -579,29 +651,28 @@ demo_file_read(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, void *buf) { DEMO_FILE_HANDLE *demo_fh; + DEMO_FILE_SYSTEM *demo_fs; WT_EXTENSION_API *wtext; size_t off; int ret = 0; demo_fh = (DEMO_FILE_HANDLE *)file_handle; - wtext = demo_fh->demo_fs->wtext; - + demo_fs = demo_fh->demo_fs; + wtext = demo_fs->wtext; off = (size_t)offset; + + LOCK_FILE_SYSTEM(session, demo_fs); + ++demo_fs->read_ops; if (off < demo_fh->size) { if (len > demo_fh->size - off) len = demo_fh->size - off; memcpy(buf, (uint8_t *)demo_fh->buf + off, len); - demo_fh->off = off + len; } else - ret = EINVAL; /* EOF */ - + ret = EIO; /* EOF */ + UNLOCK_FILE_SYSTEM(session, demo_fs); if (ret == 0) return (0); - /* - * WiredTiger should never request data past the end of a file, so - * flag an error if it does. - */ (void)wtext->err_printf(wtext, session, "%s: handle-read: failed to read %zu bytes at offset %zu: %s", demo_fh->iface.name, len, off, wtext->strerror(wtext, NULL, ret)); @@ -617,11 +688,14 @@ demo_file_size( WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *sizep) { DEMO_FILE_HANDLE *demo_fh; + DEMO_FILE_SYSTEM *demo_fs; - (void)session; /* Unused */ demo_fh = (DEMO_FILE_HANDLE *)file_handle; + demo_fs = demo_fh->demo_fs; + LOCK_FILE_SYSTEM(session, demo_fs); *sizep = (wt_off_t)demo_fh->size; + UNLOCK_FILE_SYSTEM(session, demo_fs); return (0); } @@ -640,15 +714,35 @@ demo_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) } /* - * demo_file_sync_nowait -- - * Ensure the content of the file is stable. This is a no-op in our - * memory backed file system. + * demo_buffer_resize -- + * Resize the write buffer. */ static int -demo_file_sync_nowait(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +demo_buffer_resize( + WT_SESSION *session, DEMO_FILE_HANDLE *demo_fh, wt_off_t offset) { - (void)file_handle; /* Unused */ - (void)session; /* Unused */ + DEMO_FILE_SYSTEM *demo_fs; + WT_EXTENSION_API *wtext; + size_t off; + void *p; + + demo_fs = demo_fh->demo_fs; + wtext = demo_fs->wtext; + off = (size_t)offset; + + /* Grow the buffer as necessary and clear any new space in the file. */ + if (demo_fh->bufsize >= off) + return (0); + + if ((p = realloc(demo_fh->buf, off)) == NULL) { + (void)wtext->err_printf(wtext, session, + "%s: failed to resize buffer", + demo_fh->iface.name, wtext->strerror(wtext, NULL, ENOMEM)); + return (ENOMEM); + } + memset((uint8_t *)p + demo_fh->bufsize, 0, off - demo_fh->bufsize); + demo_fh->buf = p; + demo_fh->bufsize = off; return (0); } @@ -662,33 +756,17 @@ demo_file_truncate( WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset) { DEMO_FILE_HANDLE *demo_fh; - WT_EXTENSION_API *wtext; - size_t off; + DEMO_FILE_SYSTEM *demo_fs; + int ret = 0; demo_fh = (DEMO_FILE_HANDLE *)file_handle; - wtext = demo_fh->demo_fs->wtext; - - /* - * Grow the buffer as necessary, clear any new space in the file, - * and reset the file's data length. - */ - off = (size_t)offset; - if (demo_fh->bufsize < off ) { - if ((demo_fh->buf = realloc(demo_fh->buf, off)) == NULL) { - (void)wtext->err_printf(wtext, session, - "demo_file_truncate: %s: failed to resize buffer", - demo_fh->iface.name, - wtext->strerror(wtext, NULL, ENOMEM)); - return (ENOMEM); - } - demo_fh->bufsize = off; - } - if (demo_fh->size < off) - memset((uint8_t *)demo_fh->buf + demo_fh->size, - 0, off - demo_fh->size); - demo_fh->size = off; + demo_fs = demo_fh->demo_fs; - return (0); + LOCK_FILE_SYSTEM(session, demo_fs); + if ((ret = demo_buffer_resize(session, demo_fh, offset)) == 0) + demo_fh->size = (size_t)offset; + UNLOCK_FILE_SYSTEM(session, demo_fs); + return (ret); } /* @@ -700,23 +778,32 @@ demo_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, const void *buf) { DEMO_FILE_HANDLE *demo_fh; + DEMO_FILE_SYSTEM *demo_fs; + WT_EXTENSION_API *wtext; size_t off; int ret = 0; demo_fh = (DEMO_FILE_HANDLE *)file_handle; - - /* Make sure the buffer is large enough for the write */ - if ((ret = demo_file_truncate(file_handle, session, - offset + (wt_off_t)(len + DEMO_FILE_SIZE_INCREMENT))) != 0) - return (ret); - + demo_fs = demo_fh->demo_fs; + wtext = demo_fs->wtext; off = (size_t)offset; - memcpy((uint8_t *)demo_fh->buf + off, buf, len); - if (off + len > demo_fh->size) - demo_fh->size = off + len; - demo_fh->off = off + len; - return (0); + LOCK_FILE_SYSTEM(session, demo_fs); + ++demo_fs->write_ops; + if ((ret = demo_buffer_resize(session, demo_fh, + offset + (wt_off_t)(len + DEMO_FILE_SIZE_INCREMENT))) == 0) { + memcpy((uint8_t *)demo_fh->buf + off, buf, len); + if (off + len > demo_fh->size) + demo_fh->size = off + len; + } + UNLOCK_FILE_SYSTEM(session, demo_fs); + if (ret == 0) + return (0); + + (void)wtext->err_printf(wtext, session, + "%s: handle-write: failed to write %zu bytes at offset %zu: %s", + demo_fh->iface.name, len, off, wtext->strerror(wtext, NULL, ret)); + return (ret); } /* @@ -731,7 +818,7 @@ demo_handle_remove(WT_SESSION *session, DEMO_FILE_HANDLE *demo_fh) WT_EXTENSION_API *wtext; demo_fs = demo_fh->demo_fs; - wtext = demo_fh->demo_fs->wtext; + wtext = demo_fs->wtext; if (demo_fh->ref != 0) { (void)wtext->err_printf(wtext, session, @@ -744,7 +831,6 @@ demo_handle_remove(WT_SESSION *session, DEMO_FILE_HANDLE *demo_fh) /* Clean up private information. */ free(demo_fh->buf); - demo_fh->buf = NULL; /* Clean up public information. */ free(demo_fh->iface.name); @@ -772,6 +858,8 @@ demo_handle_search(WT_FILE_SYSTEM *file_system, const char *name) return (demo_fh); } +static const char *home; + int main(void) { diff --git a/examples/c/queue_example.h b/examples/c/queue_example.h deleted file mode 100644 index 5f6674b5d1d..00000000000 --- a/examples/c/queue_example.h +++ /dev/null @@ -1,149 +0,0 @@ -/*- - * Public Domain 2014-2016 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -/* - * This is a stripped down copy of the FreeBSD queue.h include file to make - * TAILQ_XXX functionality available in WiredTiger example programs. - */ - -/*- - * Copyright (c) 1991, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)queue.h 8.5 (Berkeley) 8/20/94 - * $FreeBSD: releng/10.2/sys/sys/queue.h 279633 2015-03-05 09:23:43Z hselasky $ - */ - -/* - * Tail queue declarations. - */ -#define TAILQ_HEAD(name, type) \ -struct name { \ - struct type *tqh_first; /* first element */ \ - struct type **tqh_last; /* addr of last next element */ \ -} - -#define TAILQ_ENTRY(type) \ -struct { \ - struct type *tqe_next; /* next element */ \ - struct type **tqe_prev; /* address of previous next element */ \ -} - -/* - * Tail queue functions. - */ -#define TAILQ_EMPTY(head) ((head)->tqh_first == NULL) - -#define TAILQ_FIRST(head) ((head)->tqh_first) - -#define TAILQ_FOREACH(var, head, field) \ - for ((var) = TAILQ_FIRST((head)); \ - (var); \ - (var) = TAILQ_NEXT((var), field)) - -#define TAILQ_INIT(head) do { \ - TAILQ_FIRST((head)) = NULL; \ - (head)->tqh_last = &TAILQ_FIRST((head)); \ -} while (0) - -#define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ - if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\ - TAILQ_NEXT((elm), field)->field.tqe_prev = \ - &TAILQ_NEXT((elm), field); \ - else { \ - (head)->tqh_last = &TAILQ_NEXT((elm), field); \ - } \ - TAILQ_NEXT((listelm), field) = (elm); \ - (elm)->field.tqe_prev = &TAILQ_NEXT((listelm), field); \ -} while (0) - -#define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ - (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ - TAILQ_NEXT((elm), field) = (listelm); \ - *(listelm)->field.tqe_prev = (elm); \ - (listelm)->field.tqe_prev = &TAILQ_NEXT((elm), field); \ -} while (0) - -#define TAILQ_INSERT_HEAD(head, elm, field) do { \ - if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL) \ - TAILQ_FIRST((head))->field.tqe_prev = \ - &TAILQ_NEXT((elm), field); \ - else \ - (head)->tqh_last = &TAILQ_NEXT((elm), field); \ - TAILQ_FIRST((head)) = (elm); \ - (elm)->field.tqe_prev = &TAILQ_FIRST((head)); \ -} while (0) - -#define TAILQ_INSERT_TAIL(head, elm, field) do { \ - TAILQ_NEXT((elm), field) = NULL; \ - (elm)->field.tqe_prev = (head)->tqh_last; \ - *(head)->tqh_last = (elm); \ - (head)->tqh_last = &TAILQ_NEXT((elm), field); \ -} while (0) - -#define TAILQ_LAST(head, headname) \ - (*(((struct headname *)((head)->tqh_last))->tqh_last)) - -#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) - -#define TAILQ_PREV(elm, headname, field) \ - (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) - -#define TAILQ_REMOVE(head, elm, field) do { \ - if ((TAILQ_NEXT((elm), field)) != NULL) \ - TAILQ_NEXT((elm), field)->field.tqe_prev = \ - (elm)->field.tqe_prev; \ - else { \ - (head)->tqh_last = (elm)->field.tqe_prev; \ - } \ - *(elm)->field.tqe_prev = TAILQ_NEXT((elm), field); \ -} while (0) diff --git a/ext/datasources/helium/helium.c b/ext/datasources/helium/helium.c index 0350f9a2752..2a66c9a0ca7 100644 --- a/ext/datasources/helium/helium.c +++ b/ext/datasources/helium/helium.c @@ -2196,8 +2196,8 @@ helium_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, EMSG_ERR(wtext, session, ret, "value_format configuration: %s", wtext->strerror(wtext, session, ret)); - ws->config_bitfield = - v.len == 2 && isdigit(v.str[0]) && v.str[1] == 't'; + ws->config_bitfield = v.len == 2 && + isdigit((u_char)v.str[0]) && v.str[1] == 't'; if ((ret = config_parser->get( config_parser, "helium_o_compress", &v)) != 0) diff --git a/ext/test/kvs_bdb/kvs_bdb.c b/ext/test/kvs_bdb/kvs_bdb.c index 05c522ff41f..3d78bca1d1b 100644 --- a/ext/test/kvs_bdb/kvs_bdb.c +++ b/ext/test/kvs_bdb/kvs_bdb.c @@ -831,7 +831,7 @@ kvs_session_open_cursor(WT_DATA_SOURCE *wtds, WT_SESSION *session, goto err; } cursor->config_bitfield = - v.len == 2 && isdigit(v.str[0]) && v.str[1] == 't'; + v.len == 2 && isdigit((u_char)v.str[0]) && v.str[1] == 't'; if ((ret = writelock(wtext, session, &ds->rwlock)) != 0) goto err; diff --git a/src/async/async_worker.c b/src/async/async_worker.c index e692bc619a9..90dac557e36 100644 --- a/src/async/async_worker.c +++ b/src/async/async_worker.c @@ -216,9 +216,8 @@ __async_worker_execop(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op, __wt_cursor_set_raw_value(&asyncop->c, &val); break; case WT_AOP_NONE: - default: - WT_RET_MSG(session, EINVAL, "Unknown async optype %d\n", - op->optype); + WT_RET_MSG(session, EINVAL, + "Unknown async optype %d\n", op->optype); } return (0); } diff --git a/src/block/block_ckpt.c b/src/block/block_ckpt.c index 716121faa06..b9f0ec25d53 100644 --- a/src/block/block_ckpt.c +++ b/src/block/block_ckpt.c @@ -63,6 +63,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, */ *root_addr_sizep = 0; +#ifdef HAVE_VERBOSE if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { if (addr != NULL) { WT_ERR(__wt_scr_alloc(session, 0, &tmp)); @@ -72,6 +73,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, "%s: load-checkpoint: %s", block->name, addr == NULL ? "[Empty]" : (const char *)tmp->data)); } +#endif /* * There's a single checkpoint in the file that can be written, all of @@ -507,6 +509,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) !F_ISSET(ckpt, WT_CKPT_DELETE)) continue; +#ifdef HAVE_VERBOSE if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { if (tmp == NULL) WT_ERR(__wt_scr_alloc(session, 0, &tmp)); @@ -516,7 +519,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) "%s: delete-checkpoint: %s: %s", block->name, ckpt->name, (const char *)tmp->data)); } - +#endif /* * Find the checkpoint into which we'll roll this checkpoint's * blocks: it's the next real checkpoint in the list, and it diff --git a/src/block/block_compact.c b/src/block/block_compact.c index 24ca6632311..02862ea842f 100644 --- a/src/block/block_compact.c +++ b/src/block/block_compact.c @@ -39,12 +39,14 @@ __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) /* Restore the original allocation plan. */ __wt_block_configure_first_fit(block, false); +#ifdef HAVE_VERBOSE /* Dump the results of the compaction pass. */ if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) { __wt_spin_lock(session, &block->live_lock); ret = __block_dump_avail(session, block, false); __wt_spin_unlock(session, &block->live_lock); } +#endif return (ret); } @@ -188,6 +190,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, } __wt_spin_unlock(session, &block->live_lock); +#ifdef HAVE_VERBOSE if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) { ++block->compact_pages_reviewed; if (*skipp) @@ -195,6 +198,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, else ++block->compact_pages_written; } +#endif return (ret); } diff --git a/src/block/block_ext.c b/src/block/block_ext.c index 6d67a66be5f..0d3e7b54f17 100644 --- a/src/block/block_ext.c +++ b/src/block/block_ext.c @@ -24,7 +24,7 @@ static int __block_append(WT_SESSION_IMPL *, static int __block_ext_overlap(WT_SESSION_IMPL *, WT_BLOCK *, WT_EXTLIST *, WT_EXT **, WT_EXTLIST *, WT_EXT **); static int __block_extlist_dump( - WT_SESSION_IMPL *, const char *, WT_EXTLIST *, bool); + WT_SESSION_IMPL *, WT_BLOCK *, WT_EXTLIST *, const char *); static int __block_merge(WT_SESSION_IMPL *, WT_BLOCK *, WT_EXTLIST *, wt_off_t, wt_off_t); @@ -1227,8 +1227,7 @@ corrupted: __wt_scr_free(session, &tmp); WT_ERR(func(session, block, el, off, size)); } - if (WT_VERBOSE_ISSET(session, WT_VERB_BLOCK)) - WT_ERR(__block_extlist_dump(session, "read extlist", el, 0)); + WT_ERR(__block_extlist_dump(session, block, el, "read")); err: __wt_scr_free(session, &tmp); return (ret); @@ -1250,8 +1249,7 @@ __wt_block_extlist_write(WT_SESSION_IMPL *session, uint32_t entries; uint8_t *p; - if (WT_VERBOSE_ISSET(session, WT_VERB_BLOCK)) - WT_RET(__block_extlist_dump(session, "write extlist", el, 0)); + WT_RET(__block_extlist_dump(session, block, el, "write")); /* * Figure out how many entries we're writing -- if there aren't any @@ -1427,38 +1425,62 @@ __wt_block_extlist_free(WT_SESSION_IMPL *session, WT_EXTLIST *el) */ static int __block_extlist_dump( - WT_SESSION_IMPL *session, const char *tag, WT_EXTLIST *el, bool show_size) + WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, const char *tag) { + WT_DECL_ITEM(t1); + WT_DECL_ITEM(t2); + WT_DECL_RET; WT_EXT *ext; - WT_SIZE *szp; + uint64_t pow, sizes[64]; + u_int i; + const char *sep; - WT_RET(__wt_verbose(session, WT_VERB_BLOCK, - "%s: %s: %" PRIu64 " bytes, by offset:%s", - tag, el->name, el->bytes, el->entries == 0 ? " [Empty]" : "")); - if (el->entries == 0) + if (!block->verify_layout && !WT_VERBOSE_ISSET(session, WT_VERB_BLOCK)) return (0); - WT_EXT_FOREACH(ext, el->off) - WT_RET(__wt_verbose(session, WT_VERB_BLOCK, - "\t{%" PRIuMAX "/%" PRIuMAX "}", - (uintmax_t)ext->off, (uintmax_t)ext->size)); + WT_ERR(__wt_scr_alloc(session, 0, &t1)); + if (block->verify_layout) + WT_ERR(__wt_msg(session, + "%s extent list %s, %" PRIu32 " entries, %s bytes", + tag, el->name, el->entries, + __wt_buf_set_size(session, el->bytes, true, t1))); + else + WT_ERR(__wt_verbose(session, WT_VERB_BLOCK, + "%s extent list %s, %" PRIu32 " entries, %s bytes", + tag, el->name, el->entries, + __wt_buf_set_size(session, el->bytes, true, t1))); - if (!show_size) - return (0); + if (ret != 0 || el->entries == 0) + goto done; - WT_RET(__wt_verbose(session, WT_VERB_BLOCK, - "%s: %s: by size:%s", - tag, el->name, el->entries == 0 ? " [Empty]" : "")); - if (el->entries == 0) - return (0); + memset(sizes, 0, sizeof(sizes)); + WT_EXT_FOREACH(ext, el->off) + for (i = 9, pow = 512;; ++i, pow *= 2) + if (ext->size <= (wt_off_t)pow) { + ++sizes[i]; + break; + } + sep = "extents by bucket:"; + t1->size = 0; + WT_ERR(__wt_scr_alloc(session, 0, &t2)); + for (i = 9, pow = 512; i < WT_ELEMENTS(sizes); ++i, pow *= 2) + if (sizes[i] != 0) { + WT_ERR(__wt_buf_catfmt(session, t1, + "%s {%s: %" PRIu64 "}", + sep, + __wt_buf_set_size(session, pow, false, t2), + sizes[i])); + sep = ","; + } - WT_EXT_FOREACH(szp, el->sz) { - WT_RET(__wt_verbose(session, WT_VERB_BLOCK, - "\t{%" PRIuMAX "}", (uintmax_t)szp->size)); - WT_EXT_FOREACH_OFF(ext, szp->off) - WT_RET(__wt_verbose(session, WT_VERB_BLOCK, - "\t\t{%" PRIuMAX "/%" PRIuMAX "}", - (uintmax_t)ext->off, (uintmax_t)ext->size)); - } - return (0); + if (block->verify_layout) + WT_ERR(__wt_msg(session, "%s", (char *)t1->data)); + else + WT_ERR(__wt_verbose( + session, WT_VERB_BLOCK, "%s", (char *)t1->data)); + +done: err: + __wt_scr_free(session, &t1); + __wt_scr_free(session, &t2); + return (ret); } diff --git a/src/block/block_open.c b/src/block/block_open.c index e58bef30a6d..1603b1574e7 100644 --- a/src/block/block_open.c +++ b/src/block/block_open.c @@ -59,7 +59,7 @@ __wt_block_manager_create( session, filename, tmp->data)); WT_ERR(__wt_msg(session, "unexpected file %s found, renamed to %s", - filename, (char *)tmp->data)); + filename, (const char *)tmp->data)); break; } } diff --git a/src/block/block_vrfy.c b/src/block/block_vrfy.c index a8e59ad0af7..af58864b9dc 100644 --- a/src/block/block_vrfy.c +++ b/src/block/block_vrfy.c @@ -20,8 +20,10 @@ static int __verify_set_file_size(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *); /* The bit list ignores the first block: convert to/from a frag/offset. */ #define WT_wt_off_TO_FRAG(block, off) \ ((off) / (block)->allocsize - 1) +#ifdef HAVE_VERBOSE #define WT_FRAG_TO_OFF(block, frag) \ (((wt_off_t)(frag + 1)) * (block)->allocsize) +#endif /* * __wt_block_verify_start -- @@ -35,6 +37,14 @@ __wt_block_verify_start(WT_SESSION_IMPL *session, WT_CONFIG_ITEM cval; wt_off_t size; + /* Configuration: strict behavior on any error. */ + WT_RET(__wt_config_gets(session, cfg, "strict", &cval)); + block->verify_strict = cval.val != 0; + + /* Configuration: dump the file's layout. */ + WT_RET(__wt_config_gets(session, cfg, "dump_layout", &cval)); + block->verify_layout = cval.val != 0; + /* * Find the last checkpoint in the list: if there are none, or the only * checkpoint we have is fake, there's no work to do. Don't complain, @@ -105,9 +115,6 @@ __wt_block_verify_start(WT_SESSION_IMPL *session, */ WT_RET(__verify_last_avail(session, block, ckpt)); - /* Configuration: strict behavior on any error. */ - WT_RET(__wt_config_gets(session, cfg, "strict", &cval)); - block->verify_strict = cval.val != 0; return (0); } @@ -152,11 +159,23 @@ __verify_set_file_size(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckpt) { WT_BLOCK_CKPT *ci, _ci; WT_DECL_RET; + WT_DECL_ITEM(tmp); ci = &_ci; WT_RET(__wt_block_ckpt_init(session, ci, ckpt->name)); WT_ERR(__wt_block_buffer_to_ckpt(session, block, ckpt->raw.data, ci)); + if (block->verify_layout) { + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_msg(session, "%s: physical size %s", block->name, + __wt_buf_set_size( + session, (uint64_t)block->size, true, tmp))); + WT_ERR( + __wt_msg(session, "%s: correcting to %s checkpoint size %s", + block->name, ckpt->name, __wt_buf_set_size( + session, (uint64_t)ci->file_size, true, tmp))); + } + /* * Verify is read-only. Set the block's file size information as if we * truncated the file during checkpoint load, so references to blocks @@ -165,6 +184,7 @@ __verify_set_file_size(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckpt) block->size = block->extend_size = ci->file_size; err: __wt_block_ckpt_destroy(session, ci); + __wt_scr_free(session, &tmp); return (ret); } @@ -253,9 +273,9 @@ __wt_verify_ckpt_load( } /* - * We don't need to list of blocks on a checkpoint's avail list, but we - * read it to ensure it wasn't corrupted. We could confirm correctness - * of intermediate avail lists (that is, if they're logically the result + * We don't need the blocks on a checkpoint's avail list, but we read it + * to ensure it wasn't corrupted. We could confirm correctness of the + * intermediate avail lists (that is, if they're logically the result * of the allocations and discards to this point). We don't because the * only avail list ever used is the one for the last checkpoint, which * is separately verified by checking it against all of the blocks found @@ -443,6 +463,7 @@ __verify_filefrag_chk(WT_SESSION_IMPL *session, WT_BLOCK *block) __bit_set(block->fragfile, last); } +#ifdef HAVE_VERBOSE if (!WT_VERBOSE_ISSET(session, WT_VERB_VERIFY)) continue; @@ -450,6 +471,7 @@ __verify_filefrag_chk(WT_SESSION_IMPL *session, WT_BLOCK *block) "file range %" PRIuMAX "-%" PRIuMAX " never verified", (uintmax_t)WT_FRAG_TO_OFF(block, first), (uintmax_t)WT_FRAG_TO_OFF(block, last)); +#endif } if (count == 0) return (0); @@ -534,6 +556,7 @@ __verify_ckptfrag_chk(WT_SESSION_IMPL *session, WT_BLOCK *block) __bit_clear(block->fragckpt, last); } +#ifdef HAVE_VERBOSE if (!WT_VERBOSE_ISSET(session, WT_VERB_VERIFY)) continue; @@ -541,6 +564,7 @@ __verify_ckptfrag_chk(WT_SESSION_IMPL *session, WT_BLOCK *block) "checkpoint range %" PRIuMAX "-%" PRIuMAX " never verified", (uintmax_t)WT_FRAG_TO_OFF(block, first), (uintmax_t)WT_FRAG_TO_OFF(block, last)); +#endif } if (count == 0) diff --git a/src/btree/bt_cursor.c b/src/btree/bt_cursor.c index 4b73b76c8c8..9a57759570a 100644 --- a/src/btree/bt_cursor.c +++ b/src/btree/bt_cursor.c @@ -559,7 +559,6 @@ retry: WT_RET(__cursor_func_init(cbt, true)); ret = __cursor_row_modify(session, cbt, false); break; - WT_ILLEGAL_VALUE_ERR(session); } err: if (ret == WT_RESTART) { @@ -640,7 +639,8 @@ retry: WT_RET(__cursor_func_init(cbt, true)); break; case BTREE_COL_FIX: case BTREE_COL_VAR: - WT_ILLEGAL_VALUE_ERR(session); + WT_ERR(__wt_illegal_value(session, NULL)); + break; } err: if (ret == WT_RESTART) { @@ -718,7 +718,6 @@ retry: WT_RET(__cursor_func_init(cbt, true)); ret = __cursor_row_modify(session, cbt, true); break; - WT_ILLEGAL_VALUE_ERR(session); } err: if (ret == WT_RESTART) { @@ -809,7 +808,6 @@ retry: WT_RET(__cursor_func_init(cbt, true)); } ret = __cursor_row_modify(session, cbt, false); break; - WT_ILLEGAL_VALUE_ERR(session); } err: if (ret == WT_RESTART) { @@ -976,7 +974,6 @@ __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp) WT_RET(__wt_compare( session, a_arg->btree->collator, &a->key, &b->key, cmpp)); break; - WT_ILLEGAL_VALUE(session); } return (0); } @@ -1027,6 +1024,7 @@ __wt_btcur_equals(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp) a = (WT_CURSOR *)a_arg; b = (WT_CURSOR *)b_arg; + cmp = 0; session = (WT_SESSION_IMPL *)a->session; /* Confirm both cursors reference the same object. */ @@ -1114,7 +1112,7 @@ __cursor_truncate_fix(WT_SESSION_IMPL *session, int (*rmfunc)(WT_SESSION_IMPL *, WT_CURSOR_BTREE *, bool)) { WT_DECL_RET; - uint8_t *value; + const uint8_t *value; /* * Handle fixed-length column-store objects separately: for row-store @@ -1143,7 +1141,7 @@ retry: WT_RET(__wt_btcur_remove(start)); if ((ret = __wt_btcur_next(start, true)) != 0) break; start->compare = 0; /* Exact match */ - value = (uint8_t *)start->iface.value.data; + value = (const uint8_t *)start->iface.value.data; if (*value != 0 && (ret = rmfunc(session, start, 1)) != 0) break; diff --git a/src/btree/bt_debug.c b/src/btree/bt_debug.c index bd5970ecf86..b1579d25dc6 100644 --- a/src/btree/bt_debug.c +++ b/src/btree/bt_debug.c @@ -431,12 +431,12 @@ __debug_tree_shape_info(WT_PAGE *page) v = page->memory_footprint; if (v >= WT_GIGABYTE) snprintf(buf, sizeof(buf), - "(%p %" PRIu64 "G)", page, v / WT_GIGABYTE); + "(%p %" PRIu64 "G)", (void *)page, v / WT_GIGABYTE); else if (v >= WT_MEGABYTE) snprintf(buf, sizeof(buf), - "(%p %" PRIu64 "M)", page, v / WT_MEGABYTE); + "(%p %" PRIu64 "M)", (void *)page, v / WT_MEGABYTE); else - snprintf(buf, sizeof(buf), "(%p %" PRIu64 ")", page, v); + snprintf(buf, sizeof(buf), "(%p %" PRIu64 ")", (void *)page, v); return (buf); } @@ -636,7 +636,7 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) page = ref->page; mod = page->modify; - __dmsg(ds, "%p", page); + __dmsg(ds, "%p", (void *)page); switch (page->type) { case WT_PAGE_COL_INT: @@ -663,7 +663,8 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) } __dmsg(ds, ": %s\n", __wt_page_type_string(page->type)); - __dmsg(ds, "\t" "disk %p, entries %" PRIu32, page->dsk, entries); + __dmsg(ds, + "\t" "disk %p, entries %" PRIu32, (void *)page->dsk, entries); __dmsg(ds, ", %s", __wt_page_is_modified(page) ? "dirty" : "clean"); __dmsg(ds, ", %s", __wt_fair_islocked( session, &page->page_lock) ? "locked" : "unlocked"); @@ -894,7 +895,7 @@ __debug_page_row_leaf(WT_DBG *ds, WT_PAGE *page) /* Dump the page's K/V pairs. */ WT_ROW_FOREACH(page, rip, i) { - WT_RET(__wt_row_leaf_key(session, page, rip, key, false)); + WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); __debug_item(ds, "K", key->data, key->size); if ((cell = __wt_row_leaf_value_cell(page, rip, NULL)) == NULL) @@ -990,10 +991,10 @@ __debug_ref(WT_DBG *ds, WT_REF *ref) __dmsg(ds, "deleted"); break; case WT_REF_LOCKED: - __dmsg(ds, "locked %p", ref->page); + __dmsg(ds, "locked %p", (void *)ref->page); break; case WT_REF_MEM: - __dmsg(ds, "memory %p", ref->page); + __dmsg(ds, "memory %p", (void *)ref->page); break; case WT_REF_READING: __dmsg(ds, "reading"); @@ -1155,14 +1156,14 @@ static void __debug_item(WT_DBG *ds, const char *tag, const void *data_arg, size_t size) { size_t i; - int ch; + u_char ch; const uint8_t *data; __dmsg(ds, "\t%s%s{", tag == NULL ? "" : tag, tag == NULL ? "" : " "); for (data = data_arg, i = 0; i < size; ++i, ++data) { ch = data[0]; - if (isprint(ch)) - __dmsg(ds, "%c", ch); + if (__wt_isprint(ch)) + __dmsg(ds, "%c", (int)ch); else __debug_hex_byte(ds, data[0]); } diff --git a/src/btree/bt_discard.c b/src/btree/bt_discard.c index 9807d5bc88f..a00bb7dc2b5 100644 --- a/src/btree/bt_discard.c +++ b/src/btree/bt_discard.c @@ -82,7 +82,7 @@ __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) if (hp != NULL) __wt_errx(session, "discarded page has hazard pointer: (%p: %s, line %d)", - hp->page, hp->file, hp->line); + (void *)hp->page, hp->file, hp->line); WT_ASSERT(session, hp == NULL); } #endif diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index 118195779e9..c97e05d74a7 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -519,7 +519,6 @@ __btree_tree_open_empty(WT_SESSION_IMPL *session, bool creation) ref->state = WT_REF_DELETED; WT_ERR(__wt_row_ikey_incr(session, root, 0, "", 1, ref)); break; - WT_ILLEGAL_VALUE_ERR(session); } /* Bulk loads require a leaf page for reconciliation: create it now. */ @@ -567,7 +566,6 @@ __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) WT_RET(__wt_page_alloc( session, WT_PAGE_ROW_LEAF, 0, false, pagep)); break; - WT_ILLEGAL_VALUE(session); } return (0); } @@ -689,10 +687,9 @@ __btree_page_sizes(WT_SESSION_IMPL *session) "size (%" PRIu32 "B)", btree->allocsize); /* - * Don't let pages grow larger than a quarter of the cache, with too- - * small caches, we can end up in a situation where nothing can be - * evicted. Take care getting the cache size: with a shared cache, - * it may not have been set. + * Don't let pages grow large compared to the cache size or we can end + * up in a situation where nothing can be evicted. Take care getting + * the cache size: with a shared cache, it may not have been set. */ WT_RET(__wt_config_gets(session, cfg, "memory_page_max", &cval)); btree->maxmempage = (uint64_t)cval.val; diff --git a/src/btree/bt_io.c b/src/btree/bt_io.c index aaf906ca785..4339de6f25c 100644 --- a/src/btree/bt_io.c +++ b/src/btree/bt_io.c @@ -343,6 +343,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, * Checksum the data if the buffer isn't compressed or checksums are * configured. */ + data_cksum = true; /* -Werror=maybe-uninitialized */ switch (btree->checksum) { case CKSUM_ON: data_cksum = true; @@ -351,7 +352,6 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, data_cksum = false; break; case CKSUM_UNCOMPRESSED: - default: data_cksum = !compressed; break; } diff --git a/src/btree/bt_misc.c b/src/btree/bt_misc.c index 7f188502a0a..b6e2cc07f5a 100644 --- a/src/btree/bt_misc.c +++ b/src/btree/bt_misc.c @@ -129,19 +129,3 @@ __wt_addr_string(WT_SESSION_IMPL *session, } return (buf->data); } - -/* - * __wt_buf_set_printable -- - * Set the contents of the buffer to a printable representation of a - * byte string. - */ -const char * -__wt_buf_set_printable( - WT_SESSION_IMPL *session, const void *p, size_t size, WT_ITEM *buf) -{ - if (__wt_raw_to_esc_hex(session, p, size, buf)) { - buf->data = "[Error]"; - buf->size = strlen("[Error]"); - } - return (buf->data); -} diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c index 89d16a3f827..086500c8b2f 100644 --- a/src/btree/bt_read.c +++ b/src/btree/bt_read.c @@ -326,7 +326,7 @@ __evict_force_check(WT_SESSION_IMPL *session, WT_REF *ref) __wt_page_evict_soon(page); /* Bump the oldest ID, we're about to do some visibility checks. */ - WT_RET(__wt_txn_update_oldest(session, false)); + WT_RET(__wt_txn_update_oldest(session, 0)); /* If eviction cannot succeed, don't try. */ return (__wt_page_can_evict(session, ref, NULL)); diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index 2d7b0a0030f..7a05a883f83 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1841,8 +1841,11 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) */ for (i = 0; i < WT_SKIP_MAXDEPTH && ins_head->tail[i] == moved_ins; ++i) ; - WT_MEM_TRANSFER(page_decr, right_incr, sizeof(WT_INSERT) + - (size_t)i * sizeof(WT_INSERT *) + WT_INSERT_KEY_SIZE(moved_ins)); + WT_MEM_TRANSFER(page_decr, right_incr, + sizeof(WT_INSERT) + (size_t)i * sizeof(WT_INSERT *)); + if (type == WT_PAGE_ROW_LEAF) + WT_MEM_TRANSFER( + page_decr, right_incr, WT_INSERT_KEY_SIZE(moved_ins)); WT_MEM_TRANSFER( page_decr, right_incr, __wt_update_list_memsize(moved_ins->upd)); @@ -1951,9 +1954,6 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) /* * Update the page accounting. - * - * XXX - * If we fail to split the parent, the page's accounting will be wrong. */ __wt_cache_page_inmem_decr(session, page, page_decr); __wt_cache_page_inmem_incr(session, right, right_incr); @@ -1999,6 +1999,9 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) ins_head->tail[0]->next[0] = moved_ins; ins_head->tail[0] = moved_ins; + /* Fix up accounting for the page size. */ + __wt_cache_page_inmem_incr(session, page, page_decr); + err: if (split_ref[0] != NULL) { /* * The address was moved to the replacement WT_REF, restore it. diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index 4404069e507..da6c53aa316 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -188,7 +188,8 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) break; case WT_SYNC_CLOSE: case WT_SYNC_DISCARD: - WT_ILLEGAL_VALUE_ERR(session); + WT_ERR(__wt_illegal_value(session, NULL)); + break; } if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { @@ -273,6 +274,8 @@ err: /* On error, clear any left-over tree walk. */ int __wt_cache_op(WT_SESSION_IMPL *session, WT_CACHE_OP op) { + WT_DECL_RET; + switch (op) { case WT_SYNC_CHECKPOINT: case WT_SYNC_CLOSE: @@ -292,10 +295,12 @@ __wt_cache_op(WT_SESSION_IMPL *session, WT_CACHE_OP op) switch (op) { case WT_SYNC_CHECKPOINT: case WT_SYNC_WRITE_LEAVES: - return (__sync_file(session, op)); + ret = __sync_file(session, op); + break; case WT_SYNC_CLOSE: case WT_SYNC_DISCARD: - return (__wt_evict_file(session, op)); - WT_ILLEGAL_VALUE(session); + ret = __wt_evict_file(session, op); + break; } + return (ret); } diff --git a/src/btree/bt_vrfy.c b/src/btree/bt_vrfy.c index 531a0dc125a..0a04c387a0f 100644 --- a/src/btree/bt_vrfy.c +++ b/src/btree/bt_vrfy.c @@ -22,13 +22,13 @@ typedef struct { #define WT_VRFY_DUMP(vs) \ ((vs)->dump_address || \ - (vs)->dump_blocks || (vs)->dump_pages || (vs)->dump_shape) + (vs)->dump_blocks || (vs)->dump_layout || (vs)->dump_pages) bool dump_address; /* Configure: dump special */ bool dump_blocks; + bool dump_layout; bool dump_pages; - bool dump_shape; - - u_int depth, depth_internal[100], depth_leaf[100]; + /* Page layout information */ + uint64_t depth, depth_internal[100], depth_leaf[100]; WT_ITEM *tmp1, *tmp2, *tmp3, *tmp4; /* Temporary buffers */ } WT_VSTUFF; @@ -59,12 +59,12 @@ __verify_config(WT_SESSION_IMPL *session, const char *cfg[], WT_VSTUFF *vs) WT_RET(__wt_config_gets(session, cfg, "dump_blocks", &cval)); vs->dump_blocks = cval.val != 0; + WT_RET(__wt_config_gets(session, cfg, "dump_layout", &cval)); + vs->dump_layout = cval.val != 0; + WT_RET(__wt_config_gets(session, cfg, "dump_pages", &cval)); vs->dump_pages = cval.val != 0; - WT_RET(__wt_config_gets(session, cfg, "dump_shape", &cval)); - vs->dump_shape = cval.val != 0; - #if !defined(HAVE_DIAGNOSTIC) if (vs->dump_blocks || vs->dump_pages) WT_RET_MSG(session, ENOTSUP, @@ -112,33 +112,38 @@ __verify_config_offsets( } /* - * __verify_tree_shape -- + * __verify_layout -- * Dump the tree shape. */ static int -__verify_tree_shape(WT_SESSION_IMPL *session, WT_VSTUFF *vs) +__verify_layout(WT_SESSION_IMPL *session, WT_VSTUFF *vs) { - uint32_t total; + uint64_t total; size_t i; for (i = 0, total = 0; i < WT_ELEMENTS(vs->depth_internal); ++i) total += vs->depth_internal[i]; WT_RET(__wt_msg( - session, "Internal page tree-depth (total %" PRIu32 "):", total)); + session, "Internal page tree-depth (total %" PRIu64 "):", total)); for (i = 0; i < WT_ELEMENTS(vs->depth_internal); ++i) - if (vs->depth_internal[i] != 0) + if (vs->depth_internal[i] != 0) { WT_RET(__wt_msg(session, - "\t%03zu: %u", i, vs->depth_internal[i])); + "\t%03" WT_SIZET_FMT ": %" PRIu64, + i, vs->depth_internal[i])); + vs->depth_internal[i] = 0; + } for (i = 0, total = 0; i < WT_ELEMENTS(vs->depth_leaf); ++i) total += vs->depth_leaf[i]; WT_RET(__wt_msg( - session, "Leaf page tree-depth (total %" PRIu32 "):", total)); + session, "Leaf page tree-depth (total %" PRIu64 "):", total)); for (i = 0; i < WT_ELEMENTS(vs->depth_leaf); ++i) - if (vs->depth_leaf[i] != 0) + if (vs->depth_leaf[i] != 0) { WT_RET(__wt_msg(session, - "\t%03zu: %u", i, vs->depth_leaf[i])); - + "\t%03" WT_SIZET_FMT ": %" PRIu64, + i, vs->depth_leaf[i])); + vs->depth_leaf[i] = 0; + } return (0); } @@ -200,9 +205,11 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) /* House-keeping between checkpoints. */ __verify_checkpoint_reset(vs); - if (WT_VRFY_DUMP(vs)) + if (WT_VRFY_DUMP(vs)) { + WT_ERR(__wt_msg(session, "%s", WT_DIVIDER)); WT_ERR(__wt_msg(session, "%s: checkpoint %s", btree->dhandle->name, ckpt->name)); + } /* Load the checkpoint. */ WT_ERR(bm->checkpoint_load(bm, session, @@ -234,8 +241,8 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(ret); /* Display the tree shape. */ - if (vs->dump_shape) - WT_ERR(__verify_tree_shape(session, vs)); + if (vs->dump_layout) + WT_ERR(__verify_layout(session, vs)); } done: diff --git a/src/btree/bt_vrfy_dsk.c b/src/btree/bt_vrfy_dsk.c index 5480a25b5ec..3a6fd8261ba 100644 --- a/src/btree/bt_vrfy_dsk.c +++ b/src/btree/bt_vrfy_dsk.c @@ -298,14 +298,11 @@ __verify_dsk_row( case WT_CELL_ADDR_LEAF_NO: case WT_CELL_KEY_OVFL: case WT_CELL_VALUE_OVFL: - ret = bm->addr_invalid( - bm, session, unpack->data, unpack->size); - WT_RET_ERROR_OK(ret, EINVAL); - if (ret == EINVAL) { + if ((ret = bm->addr_invalid( + bm, session, unpack->data, unpack->size)) == EINVAL) ret = __err_cell_corrupt_or_eof( session, cell_num, tag); - goto err; - } + WT_ERR(ret); break; } diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index a631764be7e..4afcd74520f 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -639,10 +639,11 @@ __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) int level; page = cbt->ref->page; - start = stop = NULL; /* [-Wconditional-uninitialized] */ entries = 0; /* [-Wconditional-uninitialized] */ + __cursor_pos_clear(cbt); + /* If the page has disk-based entries, select from them. */ if (page->pg_row_entries != 0) { cbt->compare = 0; @@ -795,8 +796,6 @@ __wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) btree = S2BT(session); current = NULL; - __cursor_pos_clear(cbt); - if (0) { restart: /* * Discard the currently held page and restart the search from diff --git a/src/checksum/checksum.c b/src/checksum/checksum.c index 0b086753406..b6a76dacfd8 100644 --- a/src/checksum/checksum.c +++ b/src/checksum/checksum.c @@ -1103,6 +1103,7 @@ static const uint32_t g_crc_slicing[8][256] = { #endif }; +#if !defined(__powerpc64__) /* * __wt_cksum_sw -- * Return a checksum for a chunk of memory, computed in software. @@ -1171,6 +1172,7 @@ __wt_cksum_sw(const void *chunk, size_t len) #endif return (~crc); } +#endif #if (defined(__amd64) || defined(__x86_64)) /* diff --git a/src/checksum/power8/crc32_wrapper.c b/src/checksum/power8/crc32_wrapper.c index 34ac4150338..62bd3e64f5c 100644 --- a/src/checksum/power8/crc32_wrapper.c +++ b/src/checksum/power8/crc32_wrapper.c @@ -2,7 +2,7 @@ #define CRC_TABLE #include "crc32_constants.h" -#define VMX_ALIGN 16 +#define VMX_ALIGN 16U #define VMX_ALIGN_MASK (VMX_ALIGN-1) #ifdef REFLECT @@ -26,6 +26,9 @@ static unsigned int crc32_align(unsigned int crc, unsigned char *p, unsigned int __crc32_vpmsum(unsigned int crc, unsigned char *p, unsigned long len); +/* -Werror=missing-prototypes */ +unsigned int crc32_vpmsum(unsigned int crc, unsigned char *p, + unsigned long len); unsigned int crc32_vpmsum(unsigned int crc, unsigned char *p, unsigned long len) { diff --git a/src/config/config_def.c b/src/config/config_def.c index 7a75db8857d..1b656c5a0aa 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -380,9 +380,9 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_transaction_sync[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_verify[] = { { "dump_address", "boolean", NULL, NULL, NULL, 0 }, { "dump_blocks", "boolean", NULL, NULL, NULL, 0 }, + { "dump_layout", "boolean", NULL, NULL, NULL, 0 }, { "dump_offsets", "list", NULL, NULL, NULL, 0 }, { "dump_pages", "boolean", NULL, NULL, NULL, 0 }, - { "dump_shape", "boolean", NULL, NULL, NULL, 0 }, { "strict", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -976,7 +976,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "eviction_dirty_trigger=95,eviction_target=80,eviction_trigger=95" ",file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),log=(archive=,compressor=,enabled=0," - "file_max=100MB,path=,prealloc=,recover=on,zero_fill=0)," + "file_max=100MB,path=\".\",prealloc=,recover=on,zero_fill=0)," "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(json=0,on_close=0," @@ -1102,8 +1102,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { NULL, 0 }, { "WT_SESSION.verify", - "dump_address=0,dump_blocks=0,dump_offsets=,dump_pages=0," - "dump_shape=0,strict=0", + "dump_address=0,dump_blocks=0,dump_layout=0,dump_offsets=," + "dump_pages=0,strict=0", confchk_WT_SESSION_verify, 6 }, { "colgroup.meta", @@ -1177,14 +1177,15 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_extend=,file_manager=(close_handle_minimum=250," "close_idle_time=30,close_scan_interval=10),hazard_max=1000," "in_memory=0,log=(archive=,compressor=,enabled=0,file_max=100MB," - "path=,prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," - "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" - ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),use_environment=," - "use_environment_priv=0,verbose=,write_through=", + "path=\".\",prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=" + ",worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0" + ",session_max=100,session_scratch_max=2MB," + "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=0,on_close=0," + "path=\"WiredTigerStat.%d.%H\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),use_environment=,use_environment_priv=0,verbose=," + "write_through=", confchk_wiredtiger_open, 38 }, { "wiredtiger_open_all", @@ -1198,15 +1199,15 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_extend=,file_manager=(close_handle_minimum=250," "close_idle_time=30,close_scan_interval=10),hazard_max=1000," "in_memory=0,log=(archive=,compressor=,enabled=0,file_max=100MB," - "path=,prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," - "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" - ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," - "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "transaction_sync=(enabled=0,method=fsync),use_environment=," - "use_environment_priv=0,verbose=,version=(major=0,minor=0)," - "write_through=", + "path=\".\",prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=" + ",worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0" + ",session_max=100,session_scratch_max=2MB," + "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=0,on_close=0," + "path=\"WiredTigerStat.%d.%H\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0),transaction_sync=(enabled=0" + ",method=fsync),use_environment=,use_environment_priv=0,verbose=," + "version=(major=0,minor=0),write_through=", confchk_wiredtiger_open_all, 39 }, { "wiredtiger_open_basecfg", @@ -1218,7 +1219,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "eviction_dirty_trigger=95,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," - "log=(archive=,compressor=,enabled=0,file_max=100MB,path=," + "log=(archive=,compressor=,enabled=0,file_max=100MB,path=\".\"," "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" @@ -1238,7 +1239,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "eviction_dirty_trigger=95,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," - "log=(archive=,compressor=,enabled=0,file_max=100MB,path=," + "log=(archive=,compressor=,enabled=0,file_max=100MB,path=\".\"," "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" diff --git a/src/conn/api_strerror.c b/src/conn/api_strerror.c index 87864f7f4b0..edb11957556 100644 --- a/src/conn/api_strerror.c +++ b/src/conn/api_strerror.c @@ -40,8 +40,6 @@ __wt_wiredtiger_error(int error) return ("WT_RUN_RECOVERY: recovery must be run to continue"); case WT_CACHE_FULL: return ("WT_CACHE_FULL: operation would overflow cache"); - case WT_PERM_DENIED: - return ("WT_PERM_DENIED: permission denied (internal)"); } /* diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index dde3fb6930e..98267eeeb2c 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -751,6 +751,7 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) conn->extension_api.err_printf = __wt_ext_err_printf; conn->extension_api.msg_printf = __wt_ext_msg_printf; conn->extension_api.strerror = __wt_ext_strerror; + conn->extension_api.map_windows_error = __wt_ext_map_windows_error; conn->extension_api.scr_alloc = __wt_ext_scr_alloc; conn->extension_api.scr_free = __wt_ext_scr_free; conn->extension_api.collator_config = ext_collator_config; @@ -1298,7 +1299,8 @@ __conn_config_file(WT_SESSION_IMPL *session, * the next character is a hash mark, skip to the next newline. */ for (;;) { - for (*t++ = ','; --len > 0 && isspace(*++p);) + for (*t++ = ','; + --len > 0 && __wt_isspace((u_char)*++p);) ; if (len == 0) break; @@ -1517,17 +1519,14 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) * if the file does not exist. If so, then ignore the error. * XXX Ignoring the error does allow multiple read-only * connections to exist at the same time on a read-only directory. + * + * If we got an expected permission or non-existence error then skip + * the byte lock. */ - if (F_ISSET(conn, WT_CONN_READONLY)) { - /* - * If we got an expected permission or non-existence error - * then skip the byte lock. - */ - ret = __wt_map_error_rdonly(ret); - if (ret == WT_NOTFOUND || ret == WT_PERM_DENIED) { - bytelock = false; - ret = 0; - } + if (F_ISSET(conn, WT_CONN_READONLY) && + (ret == EACCES || ret == ENOENT)) { + bytelock = false; + ret = 0; } WT_ERR(ret); if (bytelock) { @@ -1567,19 +1566,16 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_OPEN_FILE_TYPE_REGULAR, is_create ? WT_OPEN_CREATE : 0, &fh); /* - * If we're read-only, check for success as well as handled errors. - * Even if we're able to open the WiredTiger file successfully, we - * do not try to lock it. The lock file test above is the only - * one we do for read-only. + * If we're read-only, check for handled errors. Even if able to open + * the WiredTiger file successfully, we do not try to lock it. The + * lock file test above is the only one we do for read-only. */ if (F_ISSET(conn, WT_CONN_READONLY)) { - ret = __wt_map_error_rdonly(ret); - if (ret == 0 || ret == WT_NOTFOUND || ret == WT_PERM_DENIED) + if (ret == EACCES || ret == ENOENT) ret = 0; WT_ERR(ret); } else { WT_ERR(ret); - /* * Lock the WiredTiger file (for backward compatibility reasons * as described above). Immediately release the lock, it's diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 4d33ac608bb..9f15db5382b 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -158,9 +158,14 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) false, 10000, WT_MILLION, &cache->evict_cond)); WT_ERR(__wt_cond_alloc(session, "eviction waiters", false, &cache->evict_waiter_cond)); + WT_ERR(__wt_spin_init(session, &cache->evict_pass_lock, "evict pass")); WT_ERR(__wt_spin_init(session, &cache->evict_queue_lock, "cache eviction queue")); WT_ERR(__wt_spin_init(session, &cache->evict_walk_lock, "cache walk")); + if ((ret = __wt_open_internal_session(conn, "evict pass", + false, WT_SESSION_NO_DATA_HANDLES, &cache->walk_session)) != 0) + WT_ERR_MSG(NULL, ret, + "Failed to create session for eviction walks"); /* Allocate the LRU eviction queue. */ cache->evict_slots = WT_EVICT_WALK_BASE + WT_EVICT_WALK_INCR; @@ -223,6 +228,14 @@ __wt_cache_stats_update(WT_SESSION_IMPL *session) WT_STAT_SET( session, stats, cache_bytes_overflow, cache->bytes_overflow); WT_STAT_SET(session, stats, cache_bytes_leaf, leaf); + + /* + * The number of files with active walks ~= number of hazard pointers + * in the walk session. Note: reading without locking. + */ + if (conn->evict_session != NULL) + WT_STAT_SET(session, stats, cache_eviction_walks_active, + conn->evict_session->nhazard); } /* @@ -235,6 +248,7 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_SESSION *wt_session; int i; conn = S2C(session); @@ -261,8 +275,12 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_cond_auto_destroy(session, &cache->evict_cond)); WT_TRET(__wt_cond_destroy(session, &cache->evict_waiter_cond)); + __wt_spin_destroy(session, &cache->evict_pass_lock); __wt_spin_destroy(session, &cache->evict_queue_lock); __wt_spin_destroy(session, &cache->evict_walk_lock); + wt_session = &cache->walk_session->iface; + if (wt_session != NULL) + WT_TRET(wt_session->close(wt_session, NULL)); for (i = 0; i < WT_EVICT_QUEUE_MAX; ++i) { __wt_spin_destroy(session, &cache->evict_queues[i].evict_lock); diff --git a/src/conn/conn_cache_pool.c b/src/conn/conn_cache_pool.c index 898bd09ece9..75ecb6b3b4a 100644 --- a/src/conn/conn_cache_pool.c +++ b/src/conn/conn_cache_pool.c @@ -309,6 +309,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) if (!F_ISSET(conn, WT_CONN_CACHE_POOL)) return (0); + F_CLR(conn, WT_CONN_CACHE_POOL); __wt_spin_lock(session, &cp->cache_pool_lock); cp_locked = true; diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index 4c2cf9a8dc2..08fb2b24468 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -544,6 +544,7 @@ __wt_conn_dhandle_discard_single( WT_DATA_HANDLE *dhandle; WT_DECL_RET; int tret; + bool set_pass_intr; dhandle = session->dhandle; @@ -562,12 +563,17 @@ __wt_conn_dhandle_discard_single( * Kludge: interrupt the eviction server in case it is holding the * handle list lock. */ - if (!F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)) - F_SET(S2C(session)->cache, WT_CACHE_CLEAR_WALKS); + set_pass_intr = false; + if (!F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)) { + set_pass_intr = true; + (void)__wt_atomic_add32(&S2C(session)->cache->pass_intr, 1); + } /* Try to remove the handle, protected by the data handle lock. */ WT_WITH_HANDLE_LIST_LOCK(session, tret = __conn_dhandle_remove(session, final)); + if (set_pass_intr) + (void)__wt_atomic_sub32(&S2C(session)->cache->pass_intr, 1); WT_TRET(tret); /* diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 5397962bc4f..1ae370ef2fa 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -544,8 +544,6 @@ restart: while (i < WT_SLOT_POOL) { save_i = i; slot = &log->slot_pool[i++]; - WT_ASSERT(session, slot->slot_state != 0 || - slot->slot_release_lsn.l.file >= log->write_lsn.l.file); if (slot->slot_state != WT_LOG_SLOT_WRITTEN) continue; written[written_i].slot_index = save_i; diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c index f5722d343f7..9c978fed843 100644 --- a/src/conn/conn_open.c +++ b/src/conn/conn_open.c @@ -93,7 +93,8 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) * transaction ID will catch up with the current ID. */ for (;;) { - WT_TRET(__wt_txn_update_oldest(session, true)); + WT_TRET(__wt_txn_update_oldest(session, + WT_TXN_OLDEST_STRICT | WT_TXN_OLDEST_WAIT)); if (txn_global->oldest_id == txn_global->current) break; __wt_yield(); diff --git a/src/cursor/cur_bulk.c b/src/cursor/cur_bulk.c index c013383fa61..d1a53057650 100644 --- a/src/cursor/cur_bulk.c +++ b/src/cursor/cur_bulk.c @@ -328,7 +328,6 @@ __wt_curbulk_init(WT_SESSION_IMPL *session, c->insert = skip_sort_check ? __curbulk_insert_row_skip_check : __curbulk_insert_row; break; - WT_ILLEGAL_VALUE(session); } cbulk->first_insert = true; diff --git a/src/cursor/cur_dump.c b/src/cursor/cur_dump.c index 32353e0a28d..595915df7b7 100644 --- a/src/cursor/cur_dump.c +++ b/src/cursor/cur_dump.c @@ -128,7 +128,7 @@ str2recno(WT_SESSION_IMPL *session, const char *p, uint64_t *recnop) * forth -- none of them are OK with us. Check the string starts with * digit, that turns off the special processing. */ - if (!isdigit(p[0])) + if (!__wt_isdigit((u_char)p[0])) goto format; errno = 0; diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index b35558e04b3..0760a07a3aa 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -170,7 +170,7 @@ __curjoin_iter_set_entry(WT_CURSOR_JOIN_ITER *iter, u_int entry_pos) iter->entry_count = 1; WT_ASSERT(iter->session, iter->entry_pos < iter->entry_count); - entry->stats.actual_count = 0; + entry->stats.iterated = 0; if (entry->subjoin == NULL) { for (topjoin = iter->cjoin; topjoin->parent != NULL; @@ -303,8 +303,7 @@ again: cursor, iter->cursor, iter->entry->repack_format, iter->entry->index != NULL)); iter->curkey = &cursor->key; - iter->entry->stats.actual_count++; - iter->entry->stats.accesses++; + iter->entry->stats.iterated++; return (0); } @@ -608,7 +607,7 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, F_ISSET(entry, WT_CURJOIN_ENTRY_DISJUNCTION)))) return (0); /* no checks to make */ - entry->stats.accesses++; + entry->stats.membership_check++; bloom_found = false; if (entry->bloom != NULL) { @@ -652,6 +651,7 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, memset(&v, 0, sizeof(v)); /* Keep lint quiet. */ c = entry->main; c->set_key(c, key); + entry->stats.main_access++; if ((ret = c->search(c)) == 0) ret = c->get_value(c, &v); else if (ret == WT_NOTFOUND) @@ -801,6 +801,7 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, collator = (entry->index == NULL) ? NULL : entry->index->collator; while (ret == 0) { WT_ERR(c->get_key(c, &curkey)); + entry->stats.iterated++; if (entry->index != NULL) { /* * Repack so it's comparable to the @@ -875,7 +876,7 @@ insert: else WT_ERR(c->get_key(c, &curvalue)); WT_ERR(__wt_bloom_insert(bloom, &curvalue)); - entry->stats.actual_count++; + entry->stats.bloom_insert++; advance: if ((ret = c->next(c)) == WT_NOTFOUND) break; @@ -1107,6 +1108,7 @@ __curjoin_next(WT_CURSOR *cursor) * A failed search is not expected, convert WT_NOTFOUND into a * generic error. */ + iter->entry->stats.main_access++; if ((ret = c->search(c)) == WT_NOTFOUND) ret = WT_ERROR; WT_ERR(ret); diff --git a/src/cursor/cur_json.c b/src/cursor/cur_json.c index 133b7b9ac9b..f0fa0d8aec2 100644 --- a/src/cursor/cur_json.c +++ b/src/cursor/cur_json.c @@ -66,7 +66,7 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, u_char *buf, size_t bufsz, WT_CONFIG_ITEM *name) { WT_PACK_VALUE *pv; - const char *p, *end; + const u_char *p, *end; size_t s, n; pv = (WT_PACK_VALUE *)voidpv; @@ -86,7 +86,7 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, case 'S': /* Account for '"' quote in front and back. */ s += 2; - p = (const char *)pv->u.s; + p = (const u_char *)pv->u.s; if (bufsz > 0) { *buf++ = '"'; bufsz--; @@ -122,7 +122,7 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, case 'U': case 'u': s += 2; - p = (const char *)pv->u.item.data; + p = (const u_char *)pv->u.item.data; end = p + pv->u.item.size; if (bufsz > 0) { *buf++ = '"'; @@ -314,14 +314,14 @@ __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) * Can be called with null buf for sizing. */ size_t -__wt_json_unpack_char(char ch, u_char *buf, size_t bufsz, bool force_unicode) +__wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) { - char abbrev; + u_char abbrev; if (!force_unicode) { - if (isprint(ch) && ch != '\\' && ch != '"') { + if (__wt_isprint(ch) && ch != '\\' && ch != '"') { if (bufsz >= 1) - *buf = (u_char)ch; + *buf = ch; return (1); } else { abbrev = '\0'; @@ -346,7 +346,7 @@ __wt_json_unpack_char(char ch, u_char *buf, size_t bufsz, bool force_unicode) if (abbrev != '\0') { if (bufsz >= 2) { *buf++ = '\\'; - *buf = (u_char)abbrev; + *buf = abbrev; } return (2); } @@ -390,7 +390,7 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, } for (nkeys = 0; *keyformat; keyformat++) - if (!isdigit(*keyformat)) + if (!__wt_isdigit((u_char)*keyformat)) nkeys++; p = beginkey; @@ -413,12 +413,13 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, #define MATCH_KEYWORD(session, in, result, keyword, matchval) do { \ size_t _kwlen = strlen(keyword); \ - if (strncmp(in, keyword, _kwlen) == 0 && !isalnum(in[_kwlen])) { \ + if (strncmp(in, keyword, _kwlen) == 0 && \ + !__wt_isalnum((u_char)in[_kwlen])) { \ in += _kwlen; \ result = matchval; \ } else { \ const char *_bad = in; \ - while (isalnum(*in)) \ + while (__wt_isalnum((u_char)*in)) \ in++; \ __wt_errx(session, "unknown keyword \"%.*s\" in JSON", \ (int)(in - _bad), _bad); \ @@ -460,7 +461,7 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, result = -1; session = (WT_SESSION_IMPL *)wt_session; - while (isspace(*src)) + while (__wt_isspace((u_char)*src)) src++; *tokstart = src; @@ -520,13 +521,12 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, isfloat = false; if (*src == '-') src++; - while ((ch = *src) != '\0' && isdigit(ch)) + while ((ch = *src) != '\0' && __wt_isdigit((u_char)ch)) src++; if (*src == '.') { isfloat = true; src++; - while ((ch = *src) != '\0' && - isdigit(ch)) + while ((ch = *src) != '\0' && __wt_isdigit((u_char)ch)) src++; } if (*src == 'e' || *src == 'E') { @@ -534,8 +534,7 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, src++; if (*src == '+' || *src == '-') src++; - while ((ch = *src) != '\0' && - isdigit(ch)) + while ((ch = *src) != '\0' && __wt_isdigit((u_char)ch)) src++; } result = isfloat ? 'f' : 'i'; @@ -560,10 +559,10 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, default: /* An illegal token, move past it anyway */ bad = src; - isalph = isalnum(*src); + isalph = __wt_isalnum((u_char)*src); src++; if (isalph) - while (*src != '\0' && isalnum(*src)) + while (*src != '\0' && __wt_isalnum((u_char)*src)) src++; __wt_errx(session, "unknown token \"%.*s\" in JSON", (int)(src - bad), bad); diff --git a/src/cursor/cur_std.c b/src/cursor/cur_std.c index 7f220a3faa2..8bb8931f36f 100644 --- a/src/cursor/cur_std.c +++ b/src/cursor/cur_std.c @@ -435,7 +435,7 @@ __wt_cursor_get_valuev(WT_CURSOR *cursor, va_list ap) } else if (WT_STREQ(fmt, "S")) *va_arg(ap, const char **) = cursor->value.data; else if (WT_STREQ(fmt, "t") || - (isdigit(fmt[0]) && WT_STREQ(fmt + 1, "t"))) + (__wt_isdigit((u_char)fmt[0]) && WT_STREQ(fmt + 1, "t"))) *va_arg(ap, uint8_t *) = *(uint8_t *)cursor->value.data; else ret = __wt_struct_unpackv(session, @@ -496,7 +496,7 @@ __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap) sz = strlen(str) + 1; buf->data = str; } else if (WT_STREQ(fmt, "t") || - (isdigit(fmt[0]) && WT_STREQ(fmt + 1, "t"))) { + (__wt_isdigit((u_char)fmt[0]) && WT_STREQ(fmt + 1, "t"))) { sz = 1; WT_ERR(__wt_buf_initsize(session, buf, sz)); *(uint8_t *)buf->mem = (uint8_t)va_arg(ap, int); diff --git a/src/cursor/cur_table.c b/src/cursor/cur_table.c index 9eb88ec6fcd..6d50523043a 100644 --- a/src/cursor/cur_table.c +++ b/src/cursor/cur_table.c @@ -972,7 +972,8 @@ __wt_curtable_open(WT_SESSION_IMPL *session, if (0) { err: if (*cursorp != NULL) { - WT_TRET(__wt_cursor_close(*cursorp)); + if (*cursorp != cursor) + WT_TRET(__wt_cursor_close(*cursorp)); *cursorp = NULL; } WT_TRET(__curtable_close(cursor)); diff --git a/src/docs/custom-file-systems.dox b/src/docs/custom-file-systems.dox index cddec0e4258..d496002b0fb 100644 --- a/src/docs/custom-file-systems.dox +++ b/src/docs/custom-file-systems.dox @@ -17,6 +17,12 @@ allocation is attempted. See the WiredTiger POSIX file system implementation for an example of how the fallocate method might be changed after initialization. +WT_FILE_SYSTEM and WT_FILE_HANDLE methods are expected to return POSIX +1003.1 or ANSI C standard error codes on failure. Custom file systems +on Windows systems can use the WT_EXTENSION_API::map_windows_error +method to translate Windows system errors into POSIX system errors for +return to WiredTiger. + WT_FILE_SYSTEM and WT_FILE_HANDLE methods which fail but not fatally (for example, a WT_FILE_HANDLE::truncate method call which fails because the file is currently mapped into memory), should return EBUSY. @@ -25,6 +31,9 @@ WT_FILE_SYSTEM and WT_FILE_HANDLE methods which fail fatally, but not in all cases (for example, a WT_FILE_HANDLE::fadvise method call which only supports ::WT_FILE_HANDLE_WILLNEED), should return ENOTSUP. +Additionally, custom file system functions may return ::WT_PANIC to +shut down the system. + Unless explicitly stated otherwise, WiredTiger may invoke methods on the WT_FILE_SYSTEM and WT_FILE_HANDLE interfaces from multiple threads concurrently. It is the responsibility of the implementation to protect diff --git a/src/evict/evict_file.c b/src/evict/evict_file.c index ffd48afd1a7..b0cd50cc655 100644 --- a/src/evict/evict_file.c +++ b/src/evict/evict_file.c @@ -26,7 +26,8 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) WT_RET(__wt_evict_file_exclusive_on(session)); /* Make sure the oldest transaction ID is up-to-date. */ - WT_RET(__wt_txn_update_oldest(session, true)); + WT_RET(__wt_txn_update_oldest( + session, WT_TXN_OLDEST_STRICT | WT_TXN_OLDEST_WAIT)); /* Walk the tree, discarding pages. */ next_ref = NULL; @@ -88,7 +89,8 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) break; case WT_SYNC_CHECKPOINT: case WT_SYNC_WRITE_LEAVES: - WT_ILLEGAL_VALUE_ERR(session); + WT_ERR(__wt_illegal_value(session, NULL)); + break; } } diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index f5a6c33e50f..8ea487bbf83 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -9,15 +9,15 @@ #include "wt_internal.h" static int __evict_clear_all_walks(WT_SESSION_IMPL *); -static int __evict_clear_walks(WT_SESSION_IMPL *); +static int __evict_helper(WT_SESSION_IMPL *); static int WT_CDECL __evict_lru_cmp(const void *, const void *); static int __evict_lru_pages(WT_SESSION_IMPL *, bool); static int __evict_lru_walk(WT_SESSION_IMPL *); static int __evict_page(WT_SESSION_IMPL *, bool); static int __evict_pass(WT_SESSION_IMPL *); +static int __evict_server(WT_SESSION_IMPL *, bool *); static int __evict_walk(WT_SESSION_IMPL *, uint32_t); static int __evict_walk_file(WT_SESSION_IMPL *, uint32_t, u_int *); -static WT_THREAD_RET __evict_worker(void *); /* * __evict_read_gen -- @@ -152,6 +152,7 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; +#ifdef HAVE_VERBOSE if (WT_VERBOSE_ISSET(session, WT_VERB_EVICTSERVER)) { uint64_t bytes_inuse, bytes_max; @@ -165,107 +166,159 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) bytes_inuse <= bytes_max ? "<=" : ">", bytes_max / WT_MEGABYTE)); } +#endif return (__wt_cond_auto_signal(session, cache->evict_cond)); } /* - * __evict_server -- - * Thread to evict pages from the cache. + * __evict_thread_run -- + * General wrapper for any eviction thread. */ static WT_THREAD_RET -__evict_server(void *arg) +__evict_thread_run(void *arg) { WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_SESSION_IMPL *session; -#ifdef HAVE_DIAGNOSTIC - struct timespec now, stuck_ts; -#endif - uint64_t pages_evicted = 0; - u_int spins; + bool did_work; session = arg; conn = S2C(session); cache = conn->cache; #ifdef HAVE_DIAGNOSTIC - WT_ERR(__wt_epoch(session, &stuck_ts)); /* -Wuninitialized */ + if (session == conn->evict_session) + WT_ERR(__wt_epoch( + session, &cache->stuck_ts)); /* -Wuninitialized */ #endif while (F_ISSET(conn, WT_CONN_EVICTION_RUN)) { - /* Evict pages from the cache as needed. */ - WT_ERR(__evict_pass(session)); - - if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) - break; - - /* - * Clear the walks so we don't pin pages while asleep, - * otherwise we can block applications evicting large pages. - */ - if (!F_ISSET(cache, WT_CACHE_STUCK)) { - for (spins = 0; (ret = __wt_spin_trylock( - session, &conn->dhandle_lock)) == EBUSY && - !F_ISSET(cache, WT_CACHE_CLEAR_WALKS); - spins++) { - if (spins < WT_THOUSAND) - __wt_yield(); - else - __wt_sleep(0, WT_THOUSAND); - } + if (conn->evict_tid_set && + __wt_spin_trylock(session, &cache->evict_pass_lock) == 0) { /* - * If we gave up acquiring the lock, that indicates a - * session is waiting for us to clear walks. Do that - * as part of a normal pass (without the handle list - * lock) to avoid deadlock. + * Cannot use WT_WITH_PASS_LOCK because this is a try + * lock. Fix when that is supported. We set the flag + * on both sessions because we may call clear_walk when + * we are walking with the walk session, locked. */ - if (ret == EBUSY) - continue; - WT_ERR(ret); - ret = __evict_clear_all_walks(session); - __wt_spin_unlock(session, &conn->dhandle_lock); + F_SET(session, WT_SESSION_LOCKED_PASS); + F_SET(cache->walk_session, WT_SESSION_LOCKED_PASS); + ret = __evict_server(session, &did_work); + F_CLR(cache->walk_session, WT_SESSION_LOCKED_PASS); + F_CLR(session, WT_SESSION_LOCKED_PASS); + __wt_spin_unlock(session, &cache->evict_pass_lock); WT_ERR(ret); + WT_ERR(__wt_verbose( + session, WT_VERB_EVICTSERVER, "sleeping")); + /* Don't rely on signals: check periodically. */ + WT_ERR(__wt_cond_auto_wait( + session, cache->evict_cond, did_work)); + WT_ERR(__wt_verbose( + session, WT_VERB_EVICTSERVER, "waking")); + } else + WT_ERR(__evict_helper(session)); + } - /* Next time we wake up, reverse the sweep direction. */ - cache->flags ^= WT_CACHE_WALK_REVERSE; - pages_evicted = 0; - } else if (pages_evicted != cache->pages_evict) { - pages_evicted = cache->pages_evict; + if (session == conn->evict_session) { + /* + * The eviction server is shutting down: in case any trees are + * still open, clear all walks now so that they can be closed. + */ + WT_WITH_PASS_LOCK(session, ret, + ret = __evict_clear_all_walks(session)); + WT_ERR(ret); + } + WT_ERR(__wt_verbose( + session, WT_VERB_EVICTSERVER, "cache eviction thread exiting")); + + /* + * The only two cases when eviction workers are expected to stop are + * when recovery is finished or when the connection is closing. Check + * otherwise fewer eviction worker threads may be running than + * expected. + */ + WT_ASSERT(session, F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_RECOVERING)); + if (0) { +err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); + } + return (WT_THREAD_RET_VALUE); +} + +/* + * __evict_server -- + * Thread to evict pages from the cache. + */ +static int +__evict_server(WT_SESSION_IMPL *session, bool *did_work) +{ + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; #ifdef HAVE_DIAGNOSTIC - WT_ERR(__wt_epoch(session, &stuck_ts)); - } else { - /* After being stuck for 5 minutes, give up. */ - WT_ERR(__wt_epoch(session, &now)); - if (WT_TIMEDIFF_SEC(now, stuck_ts) > 300) { - __wt_err(session, ETIMEDOUT, - "Cache stuck for too long, giving up"); - (void)__wt_cache_dump(session, NULL); - WT_ERR(ETIMEDOUT); - } + struct timespec now; #endif - } + uint64_t orig_pages_evicted; + u_int spins; - WT_ERR(__wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping")); - /* Don't rely on signals: check periodically. */ - WT_ERR(__wt_cond_auto_wait( - session, cache->evict_cond, pages_evicted != 0)); - WT_ERR(__wt_verbose(session, WT_VERB_EVICTSERVER, "waking")); - } + conn = S2C(session); + cache = conn->cache; + WT_ASSERT(session, did_work != NULL); + *did_work = false; + orig_pages_evicted = cache->pages_evicted; + + /* Evict pages from the cache as needed. */ + WT_RET(__evict_pass(session)); + + if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) + return (0); /* - * The eviction server is shutting down: in case any trees are still - * open, clear all walks now so that they can be closed. + * Clear the walks so we don't pin pages while asleep, + * otherwise we can block applications evicting large pages. */ - WT_ERR(__evict_clear_all_walks(session)); - - WT_ERR(__wt_verbose( - session, WT_VERB_EVICTSERVER, "cache eviction server exiting")); + if (!F_ISSET(cache, WT_CACHE_STUCK)) { + for (spins = 0; (ret = __wt_spin_trylock( + session, &conn->dhandle_lock)) == EBUSY && + cache->pass_intr == 0; spins++) { + if (spins < WT_THOUSAND) + __wt_yield(); + else + __wt_sleep(0, WT_THOUSAND); + } + /* + * If we gave up acquiring the lock, that indicates a + * session is waiting for us to clear walks. Do that + * as part of a normal pass (without the handle list + * lock) to avoid deadlock. + */ + if (ret == EBUSY) + return (0); + WT_RET(ret); + ret = __evict_clear_all_walks(session); + __wt_spin_unlock(session, &conn->dhandle_lock); + WT_RET(ret); - if (0) { -err: WT_PANIC_MSG(session, ret, "cache eviction server error"); + /* Next time we wake up, reverse the sweep direction. */ + cache->flags ^= WT_CACHE_WALK_REVERSE; + cache->pages_evicted = 0; + } else if (cache->pages_evicted != cache->pages_evict) { + cache->pages_evicted = cache->pages_evict; +#ifdef HAVE_DIAGNOSTIC + WT_RET(__wt_epoch(session, &cache->stuck_ts)); + } else { + /* After being stuck for 5 minutes, give up. */ + WT_RET(__wt_epoch(session, &now)); + if (WT_TIMEDIFF_SEC(now, cache->stuck_ts) > 300) { + __wt_err(session, ETIMEDOUT, + "Cache stuck for too long, giving up"); + (void)__wt_cache_dump(session, NULL); + WT_RET(ETIMEDOUT); + } +#endif } - return (WT_THREAD_RET_VALUE); + *did_work = cache->pages_evicted != orig_pages_evicted; + return (0); } /* @@ -284,6 +337,7 @@ __evict_workers_resize(WT_SESSION_IMPL *session) uint32_t i, session_flags; conn = S2C(session); + workers = NULL; /* -Wconditional-uninitialized */ if (conn->evict_workers_alloc < conn->evict_workers_max) { alloc = conn->evict_workers_alloc * sizeof(*workers); @@ -315,7 +369,8 @@ __evict_workers_resize(WT_SESSION_IMPL *session) ++conn->evict_workers; F_SET(&workers[i], WT_EVICT_WORKER_RUN); WT_ERR(__wt_thread_create(workers[i].session, - &workers[i].tid, __evict_worker, &workers[i])); + &workers[i].tid, __evict_thread_run, + workers[i].session)); } } @@ -369,7 +424,7 @@ __wt_evict_create(WT_SESSION_IMPL *session) * the worker's sessions are created. */ WT_RET(__wt_thread_create( - session, &conn->evict_tid, __evict_server, session)); + session, &conn->evict_tid, __evict_thread_run, session)); conn->evict_tid_set = true; return (0); @@ -439,39 +494,22 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) } /* - * __evict_worker -- + * __evict_helper -- * Thread to help evict pages from the cache. */ -static WT_THREAD_RET -__evict_worker(void *arg) +static int +__evict_helper(WT_SESSION_IMPL *session) { WT_CACHE *cache; - WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_EVICT_WORKER *worker; - WT_SESSION_IMPL *session; - worker = arg; - session = worker->session; - conn = S2C(session); - cache = conn->cache; - - while (F_ISSET(conn, WT_CONN_EVICTION_RUN) && - F_ISSET(worker, WT_EVICT_WORKER_RUN)) { - /* Don't spin in a busy loop if there is no work to do */ - if ((ret = __evict_lru_pages(session, false)) == WT_NOTFOUND) - WT_ERR(__wt_cond_wait( - session, cache->evict_waiter_cond, 10000)); - else - WT_ERR(ret); - } - WT_ERR(__wt_verbose( - session, WT_VERB_EVICTSERVER, "cache eviction worker exiting")); - - if (0) { -err: WT_PANIC_MSG(session, ret, "cache eviction worker error"); - } - return (WT_THREAD_RET_VALUE); + cache = S2C(session)->cache; + if ((ret = __evict_lru_pages(session, false)) == WT_NOTFOUND) + WT_RET(__wt_cond_wait( + session, cache->evict_waiter_cond, 10000)); + else + WT_RET(ret); + return (0); } /* @@ -579,12 +617,8 @@ __evict_pass(WT_SESSION_IMPL *session) * If there is a request to clear eviction walks, do that now, * before checking if the cache is full. */ - if (F_ISSET(cache, WT_CACHE_CLEAR_WALKS)) { - F_CLR(cache, WT_CACHE_CLEAR_WALKS); - WT_RET(__evict_clear_walks(session)); - WT_RET(__wt_cond_signal( - session, cache->evict_waiter_cond)); - } + if (cache->pass_intr != 0) + break; /* * Increment the shared read generation. Do this occasionally @@ -602,9 +636,10 @@ __evict_pass(WT_SESSION_IMPL *session) * * Do this every time the eviction server wakes up, regardless * of whether the cache is full, to prevent the oldest ID - * falling too far behind. + * falling too far behind. Don't wait to lock the table: with + * highly threaded workloads, that creates a bottleneck. */ - WT_RET(__wt_txn_update_oldest(session, loop > 0)); + WT_RET(__wt_txn_update_oldest(session, WT_TXN_OLDEST_STRICT)); if (!__evict_update_work(session)) break; @@ -630,7 +665,7 @@ __evict_pass(WT_SESSION_IMPL *session) worker = &conn->evict_workctx[conn->evict_workers++]; F_SET(worker, WT_EVICT_WORKER_RUN); WT_RET(__wt_thread_create(session, - &worker->tid, __evict_worker, worker)); + &worker->tid, __evict_thread_run, worker->session)); } WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, @@ -648,7 +683,7 @@ __evict_pass(WT_SESSION_IMPL *session) */ if (pages_evicted == cache->pages_evict) { WT_STAT_FAST_CONN_INCR(session, - cache_eviction_server_slept); + cache_eviction_server_slept); /* * Back off if we aren't making progress: walks hold * the handle list lock, which blocks other operations @@ -689,11 +724,13 @@ __evict_clear_walk(WT_SESSION_IMPL *session) { WT_BTREE *btree; WT_CACHE *cache; + WT_DECL_RET; WT_REF *ref; btree = S2BT(session); cache = S2C(session)->cache; + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_PASS)); if (session->dhandle == cache->evict_file_next) cache->evict_file_next = NULL; @@ -705,30 +742,9 @@ __evict_clear_walk(WT_SESSION_IMPL *session) * assert we never try to evict the current eviction walk point). */ btree->evict_ref = NULL; - return (__wt_page_release(session, ref, WT_READ_NO_EVICT)); -} - -/* - * __evict_clear_walks -- - * Clear the eviction walk points for any file a session is waiting on. - */ -static int -__evict_clear_walks(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - WT_SESSION_IMPL *s; - u_int i, session_cnt; - - conn = S2C(session); - - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (s = conn->sessions, i = 0; i < session_cnt; ++s, ++i) { - if (!s->active || !F_ISSET(s, WT_SESSION_CLEAR_EVICT_WALK)) - continue; - WT_WITH_DHANDLE( - session, s->dhandle, WT_TRET(__evict_clear_walk(session))); - } + WT_WITH_DHANDLE(cache->walk_session, session->dhandle, + (ret = __wt_page_release(cache->walk_session, + ref, WT_READ_NO_EVICT))); return (ret); } @@ -753,39 +769,6 @@ __evict_clear_all_walks(WT_SESSION_IMPL *session) } /* - * __evict_request_clear_walk -- - * Request that the eviction server clear the tree's current eviction - * point. - */ -static int -__evict_request_clear_walk(WT_SESSION_IMPL *session) -{ - WT_BTREE *btree; - WT_CACHE *cache; - WT_DECL_RET; - - btree = S2BT(session); - cache = S2C(session)->cache; - - F_SET(session, WT_SESSION_CLEAR_EVICT_WALK); - - while (ret == 0 && (btree->evict_ref != NULL || - cache->evict_file_next == session->dhandle)) { - F_SET(cache, WT_CACHE_CLEAR_WALKS); - ret = __wt_cond_wait( - session, cache->evict_waiter_cond, 100000); - } - - F_CLR(session, WT_SESSION_CLEAR_EVICT_WALK); - - /* An error is unexpected - flag the failure. */ - if (ret != 0) - __wt_err(session, ret, "Failed to clear eviction walk point"); - - return (ret); -} - -/* * __wt_evict_file_exclusive_on -- * Get exclusive eviction access to a file and discard any of the file's * blocks queued for eviction. @@ -822,10 +805,14 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) * this point. */ F_SET(btree, WT_BTREE_NO_EVICTION); + (void)__wt_atomic_add32(&cache->pass_intr, 1); WT_FULL_BARRIER(); /* Clear any existing LRU eviction walk for the file. */ - WT_ERR(__evict_request_clear_walk(session)); + WT_WITH_PASS_LOCK(session, ret, + ret = __evict_clear_walk(session)); + (void)__wt_atomic_sub32(&cache->pass_intr, 1); + WT_ERR(ret); /* * The eviction candidate list might reference pages from the file, @@ -949,7 +936,7 @@ __evict_lru_walk(WT_SESSION_IMPL *session) queue_index = cache->evict_queue_fill++ % WT_EVICT_QUEUE_MAX; evict_queue = &cache->evict_queues[queue_index]; /* Get some more pages to consider for eviction. */ - if ((ret = __evict_walk(session, queue_index)) != 0) + if ((ret = __evict_walk(cache->walk_session, queue_index)) != 0) return (ret == EBUSY ? 0 : ret); /* Sort the list into LRU order and restart. */ @@ -1063,8 +1050,8 @@ __evict_lru_walk(WT_SESSION_IMPL *session) __wt_spin_unlock(session, &cache->evict_queue_lock); /* - * The eviction server thread doesn't do any actual eviction if there - * are multiple eviction workers running. + * Signal any application or helper threads that may be waiting + * to help with eviction. */ WT_RET(__wt_cond_signal(session, cache->evict_waiter_cond)); @@ -1108,7 +1095,7 @@ retry: while (slot < max_entries && ret == 0) { * If another thread is waiting on the eviction server to clear * the walk point in a tree, give up. */ - if (F_ISSET(cache, WT_CACHE_CLEAR_WALKS)) + if (cache->pass_intr != 0) break; /* @@ -1118,7 +1105,7 @@ retry: while (slot < max_entries && ret == 0) { if (!dhandle_locked) { for (spins = 0; (ret = __wt_spin_trylock( session, &conn->dhandle_lock)) == EBUSY && - !F_ISSET(cache, WT_CACHE_CLEAR_WALKS); + cache->pass_intr == 0; spins++) { if (spins < WT_THOUSAND) __wt_yield(); @@ -1244,7 +1231,7 @@ retry: while (slot < max_entries && ret == 0) { * Try two passes through all the files, give up when we have some * candidates and we aren't finding more. */ - if (!F_ISSET(cache, WT_CACHE_CLEAR_WALKS) && ret == 0 && + if (cache->pass_intr == 0 && ret == 0 && slot < max_entries && (retries < 2 || (retries < 10 && !FLD_ISSET(cache->state, WT_EVICT_PASS_WOULD_BLOCK) && @@ -1298,7 +1285,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, uint32_t queue_index, u_int *slotp) WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_REF *ref; - uint64_t pages_walked; + uint64_t pages_seen, refs_walked; uint32_t walk_flags; int internal_pages, restarts; bool enough, modified; @@ -1333,17 +1320,21 @@ __evict_walk_file(WT_SESSION_IMPL *session, uint32_t queue_index, u_int *slotp) * Once we hit the page limit, do one more step through the walk in * case we are appending and only the last page in the file is live. */ - for (evict = start, pages_walked = 0; + for (evict = start, pages_seen = refs_walked = 0; evict < end && !enough && (ret == 0 || ret == WT_NOTFOUND); ret = __wt_tree_walk_count( - session, &btree->evict_ref, &pages_walked, walk_flags)) { - enough = pages_walked > cache->evict_max_refs_per_file; + session, &btree->evict_ref, &refs_walked, walk_flags)) { + enough = refs_walked > cache->evict_max_refs_per_file; if ((ref = btree->evict_ref) == NULL) { if (++restarts == 2 || enough) break; + WT_STAT_FAST_CONN_INCR( + session, cache_eviction_walks_started); continue; } + ++pages_seen; + /* Ignore root pages entirely. */ if (__wt_ref_is_root(ref)) continue; @@ -1371,9 +1362,13 @@ __evict_walk_file(WT_SESSION_IMPL *session, uint32_t queue_index, u_int *slotp) } /* Pages we no longer need (clean or dirty), are found money. */ + if (page->read_gen == WT_READGEN_OLDEST) { + WT_STAT_FAST_CONN_INCR( + session, cache_eviction_pages_queued_oldest); + goto fast; + } if (__wt_page_is_empty(page) || - F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || - page->read_gen == WT_READGEN_OLDEST) + F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) goto fast; /* Skip clean pages if appropriate. */ @@ -1439,24 +1434,31 @@ fast: /* If the page can't be evicted, give up. */ WT_RET_NOTFOUND_OK(ret); *slotp += (u_int)(evict - start); + WT_STAT_FAST_CONN_INCRV( + session, cache_eviction_pages_queued, (u_int)(evict - start)); /* * If we happen to end up on the root page, clear it. We have to track * hazard pointers, and the root page complicates that calculation. * + * Likewise if we found no new candidates during the walk: there is no + * point keeping a page pinned, since it may be the only candidate in an + * idle tree. + * * If we land on a page requiring forced eviction, move on to the next * page: we want this page evicted as quickly as possible. */ if ((ref = btree->evict_ref) != NULL) { - if (__wt_ref_is_root(ref)) + if (__wt_ref_is_root(ref) || evict == start) WT_RET(__evict_clear_walk(session)); else if (ref->page->read_gen == WT_READGEN_OLDEST) WT_RET_NOTFOUND_OK(__wt_tree_walk_count( session, &btree->evict_ref, - &pages_walked, walk_flags)); + &refs_walked, walk_flags)); } - WT_STAT_FAST_CONN_INCRV(session, cache_eviction_walk, pages_walked); + WT_STAT_FAST_CONN_INCRV(session, cache_eviction_walk, refs_walked); + WT_STAT_FAST_CONN_INCRV(session, cache_eviction_pages_seen, pages_seen); return (0); } @@ -1541,6 +1543,10 @@ __evict_get_ref( for (;;) { if (__wt_spin_trylock(session, &evict_queue->evict_lock) == 0) break; + if (!F_ISSET(session, WT_SESSION_INTERNAL)) { + __wt_spin_unlock(session, &cache->evict_queue_lock); + return (WT_NOTFOUND); + } __wt_yield(); } /* @@ -1638,6 +1644,9 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) cache->worker_evicts++; } } else { + if (__wt_page_is_modified(ref->page)) + WT_STAT_FAST_CONN_INCR( + session, cache_eviction_app_dirty); WT_STAT_FAST_CONN_INCR(session, cache_eviction_app); cache->app_evicts++; } @@ -1727,8 +1736,9 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) } /* See if eviction is still needed. */ - if (!__wt_eviction_needed(session, NULL) || - cache->pages_evict > init_evict_count + max_pages_evicted) + if (!__wt_eviction_needed(session, &pct_full) || + (pct_full < 100 && + cache->pages_evict > init_evict_count + max_pages_evicted)) return (0); /* Evict a page. */ diff --git a/src/evict/evict_page.c b/src/evict/evict_page.c index 2d20f53e9ae..305b81fe69e 100644 --- a/src/evict/evict_page.c +++ b/src/evict/evict_page.c @@ -420,7 +420,8 @@ __evict_review( * fallen behind current. */ if (modified) - WT_RET(__wt_txn_update_oldest(session, false)); + WT_RET(__wt_txn_update_oldest( + session, WT_TXN_OLDEST_STRICT)); if (!__wt_page_can_evict(session, ref, inmem_splitp)) return (EBUSY); diff --git a/src/include/block.h b/src/include/block.h index 9f652ceddb9..a8080c1651c 100644 --- a/src/include/block.h +++ b/src/include/block.h @@ -261,6 +261,7 @@ struct __wt_block { /* Verification support */ bool verify; /* If performing verification */ + bool verify_layout; /* Print out file layout information */ bool verify_strict; /* Fail hard on any error */ wt_off_t verify_size; /* Checkpoint's file size */ WT_EXTLIST verify_alloc; /* Verification allocation list */ diff --git a/src/include/btree_cmp.i b/src/include/btree_cmp.i index 1993c1be293..23a462e4e50 100644 --- a/src/include/btree_cmp.i +++ b/src/include/btree_cmp.i @@ -52,8 +52,8 @@ __wt_lex_compare(const WT_ITEM *user_item, const WT_ITEM *tree_item) for (; len > 0; len -= WT_VECTOR_SIZE, userp += WT_VECTOR_SIZE, treep += WT_VECTOR_SIZE) { - u = _mm_load_si128((__m128i *)userp); - t = _mm_load_si128((__m128i *)treep); + u = _mm_load_si128((const __m128i *)userp); + t = _mm_load_si128((const __m128i *)treep); res_eq = _mm_cmpeq_epi8(u, t); if (_mm_movemask_epi8(res_eq) != 65535) break; @@ -62,8 +62,8 @@ __wt_lex_compare(const WT_ITEM *user_item, const WT_ITEM *tree_item) for (; len > 0; len -= WT_VECTOR_SIZE, userp += WT_VECTOR_SIZE, treep += WT_VECTOR_SIZE) { - u = _mm_loadu_si128((__m128i *)userp); - t = _mm_loadu_si128((__m128i *)treep); + u = _mm_loadu_si128((const __m128i *)userp); + t = _mm_loadu_si128((const __m128i *)treep); res_eq = _mm_cmpeq_epi8(u, t); if (_mm_movemask_epi8(res_eq) != 65535) break; @@ -123,8 +123,8 @@ __wt_lex_compare_skip( tsz = tree_item->size; len = WT_MIN(usz, tsz) - *matchp; - userp = (uint8_t *)user_item->data + *matchp; - treep = (uint8_t *)tree_item->data + *matchp; + userp = (const uint8_t *)user_item->data + *matchp; + treep = (const uint8_t *)tree_item->data + *matchp; #ifdef HAVE_X86INTRIN_H /* Use vector instructions if we'll execute at least 2 of them. */ @@ -139,8 +139,8 @@ __wt_lex_compare_skip( len -= WT_VECTOR_SIZE, userp += WT_VECTOR_SIZE, treep += WT_VECTOR_SIZE, *matchp += WT_VECTOR_SIZE) { - u = _mm_load_si128((__m128i *)userp); - t = _mm_load_si128((__m128i *)treep); + u = _mm_load_si128((const __m128i *)userp); + t = _mm_load_si128((const __m128i *)treep); res_eq = _mm_cmpeq_epi8(u, t); if (_mm_movemask_epi8(res_eq) != 65535) break; @@ -150,8 +150,8 @@ __wt_lex_compare_skip( len -= WT_VECTOR_SIZE, userp += WT_VECTOR_SIZE, treep += WT_VECTOR_SIZE, *matchp += WT_VECTOR_SIZE) { - u = _mm_loadu_si128((__m128i *)userp); - t = _mm_loadu_si128((__m128i *)treep); + u = _mm_loadu_si128((const __m128i *)userp); + t = _mm_loadu_si128((const __m128i *)treep); res_eq = _mm_cmpeq_epi8(u, t); if (_mm_movemask_epi8(res_eq) != 65535) break; diff --git a/src/include/cache.h b/src/include/cache.h index 4f7981a5df9..f4a35de7201 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -76,6 +76,7 @@ struct __wt_cache { uint64_t bytes_overflow; /* Bytes of overflow pages */ uint64_t bytes_evict; /* Bytes/pages discarded by eviction */ uint64_t pages_evict; + uint64_t pages_evicted; /* Pages evicted during a pass */ uint64_t bytes_dirty; /* Bytes/pages currently dirty */ uint64_t pages_dirty; uint64_t bytes_read; /* Bytes read into memory */ @@ -86,6 +87,9 @@ struct __wt_cache { uint64_t worker_evicts; /* Pages evicted by worker threads */ uint64_t evict_max_page_size; /* Largest page seen at eviction */ +#ifdef HAVE_DIAGNOSTIC + struct timespec stuck_ts; /* Stuck timestamp */ +#endif /* * Read information. @@ -112,6 +116,8 @@ struct __wt_cache { /* * LRU eviction list information. */ + WT_SPINLOCK evict_pass_lock; /* Eviction pass lock */ + WT_SESSION_IMPL *walk_session; /* Eviction pass session */ WT_SPINLOCK evict_queue_lock; /* Eviction current queue lock */ WT_EVICT_QUEUE evict_queues[WT_EVICT_QUEUE_MAX]; WT_EVICT_QUEUE *evict_current_queue;/* LRU current queue in use */ @@ -144,19 +150,28 @@ struct __wt_cache { #define WT_EVICT_PASS_DIRTY 0x04 #define WT_EVICT_PASS_WOULD_BLOCK 0x08 uint32_t state; + /* + * Pass interrupt counter. + */ + uint32_t pass_intr; /* Interrupt eviction pass. */ /* * Flags. */ #define WT_CACHE_POOL_MANAGER 0x01 /* The active cache pool manager */ #define WT_CACHE_POOL_RUN 0x02 /* Cache pool thread running */ -#define WT_CACHE_CLEAR_WALKS 0x04 /* Clear eviction walks */ -#define WT_CACHE_STUCK 0x08 /* Eviction server is stuck */ -#define WT_CACHE_WALK_REVERSE 0x10 /* Scan backwards for candidates */ -#define WT_CACHE_WOULD_BLOCK 0x20 /* Pages that would block apps */ +#define WT_CACHE_STUCK 0x04 /* Eviction server is stuck */ +#define WT_CACHE_WALK_REVERSE 0x08 /* Scan backwards for candidates */ +#define WT_CACHE_WOULD_BLOCK 0x10 /* Pages that would block apps */ uint32_t flags; }; +#define WT_WITH_PASS_LOCK(session, ret, op) do { \ + WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_PASS)); \ + WT_WITH_LOCK(session, ret, \ + &cache->evict_pass_lock, WT_SESSION_LOCKED_PASS, op); \ +} while (0) + /* * WT_CACHE_POOL -- * A structure that represents a shared cache. diff --git a/src/include/cell.i b/src/include/cell.i index 481d2a29764..c130768e595 100644 --- a/src/include/cell.i +++ b/src/include/cell.i @@ -183,9 +183,9 @@ __wt_cell_pack_addr(WT_CELL *cell, u_int cell_type, uint64_t recno, size_t size) p = cell->__chunk + 1; if (recno == WT_RECNO_OOB) - cell->__chunk[0] = cell_type; /* Type */ + cell->__chunk[0] = (uint8_t)cell_type; /* Type */ else { - cell->__chunk[0] = cell_type | WT_CELL_64V; + cell->__chunk[0] = (uint8_t)(cell_type | WT_CELL_64V); (void)__wt_vpack_uint(&p, 0, recno); /* Record number */ } (void)__wt_vpack_uint(&p, 0, (uint64_t)size); /* Length */ @@ -207,8 +207,8 @@ __wt_cell_pack_data(WT_CELL *cell, uint64_t rle, size_t size) */ if (rle < 2 && size <= WT_CELL_SHORT_MAX) { byte = (uint8_t)size; /* Type + length */ - cell->__chunk[0] = - (byte << WT_CELL_SHORT_SHIFT) | WT_CELL_VALUE_SHORT; + cell->__chunk[0] = (uint8_t) + ((byte << WT_CELL_SHORT_SHIFT) | WT_CELL_VALUE_SHORT); return (1); } @@ -331,8 +331,8 @@ __wt_cell_pack_int_key(WT_CELL *cell, size_t size) /* Short keys have 6 bits of data length in the descriptor byte. */ if (size <= WT_CELL_SHORT_MAX) { byte = (uint8_t)size; - cell->__chunk[0] = - (byte << WT_CELL_SHORT_SHIFT) | WT_CELL_KEY_SHORT; + cell->__chunk[0] = (uint8_t) + ((byte << WT_CELL_SHORT_SHIFT) | WT_CELL_KEY_SHORT); return (1); } @@ -358,14 +358,14 @@ __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size) if (size <= WT_CELL_SHORT_MAX) { if (prefix == 0) { byte = (uint8_t)size; /* Type + length */ - cell->__chunk[0] = - (byte << WT_CELL_SHORT_SHIFT) | WT_CELL_KEY_SHORT; + cell->__chunk[0] = (uint8_t) + ((byte << WT_CELL_SHORT_SHIFT) | WT_CELL_KEY_SHORT); return (1); } else { byte = (uint8_t)size; /* Type + length */ - cell->__chunk[0] = - (byte << WT_CELL_SHORT_SHIFT) | - WT_CELL_KEY_SHORT_PFX; + cell->__chunk[0] = (uint8_t) + ((byte << WT_CELL_SHORT_SHIFT) | + WT_CELL_KEY_SHORT_PFX); cell->__chunk[1] = prefix; /* Prefix */ return (2); } @@ -585,8 +585,8 @@ restart: WT_CELL_LEN_CHK(cell, 0); unpack->cell = cell; unpack->v = 0; - unpack->raw = __wt_cell_type_raw(cell); - unpack->type = __wt_cell_type(cell); + unpack->raw = (uint8_t)__wt_cell_type_raw(cell); + unpack->type = (uint8_t)__wt_cell_type(cell); unpack->ovfl = 0; /* diff --git a/src/include/ctype.i b/src/include/ctype.i new file mode 100644 index 00000000000..b4a1ad9f318 --- /dev/null +++ b/src/include/ctype.i @@ -0,0 +1,69 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include <ctype.h> + +/* + * __wt_isalnum -- + * Wrap the ctype function without sign extension. + */ +static inline bool +__wt_isalnum(u_char c) +{ + return (isalnum(c) != 0); +} + +/* + * __wt_isalpha -- + * Wrap the ctype function without sign extension. + */ +static inline bool +__wt_isalpha(u_char c) +{ + return (isalpha(c) != 0); +} + +/* + * __wt_isdigit -- + * Wrap the ctype function without sign extension. + */ +static inline bool +__wt_isdigit(u_char c) +{ + return (isdigit(c) != 0); +} + +/* + * __wt_isprint -- + * Wrap the ctype function without sign extension. + */ +static inline bool +__wt_isprint(u_char c) +{ + return (isprint(c) != 0); +} + +/* + * __wt_isspace -- + * Wrap the ctype function without sign extension. + */ +static inline bool +__wt_isspace(u_char c) +{ + return (isspace(c) != 0); +} + +/* + * __wt_tolower -- + * Wrap the ctype function without sign extension. + */ +static inline u_char +__wt_tolower(u_char c) +{ + return ((u_char)tolower(c)); +} diff --git a/src/include/extern.h b/src/include/extern.h index 4ca5c8461a0..b0c0f6eccad 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -144,7 +144,6 @@ extern const char *__wt_page_type_string(u_int type); extern const char *__wt_cell_type_string(uint8_t type); extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, WT_REF *ref, WT_ITEM *buf); extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf); -extern const char *__wt_buf_set_printable( WT_SESSION_IMPL *session, const void *p, size_t size, WT_ITEM *buf); extern int __wt_ovfl_read(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack, WT_ITEM *store); extern int __wt_ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, void *cookie, WT_CELL_UNPACK *vpack); extern int __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell); @@ -291,7 +290,7 @@ extern int __wt_curjoin_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSO extern int __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_INDEX *idx, WT_CURSOR *ref_cursor, uint8_t flags, uint8_t range, uint64_t count, uint32_t bloom_bit_count, uint32_t bloom_hash_count); extern int __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, WT_CURSOR_JSON *json, bool iskey, va_list ap); extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor); -extern size_t __wt_json_unpack_char(char ch, u_char *buf, size_t bufsz, bool force_unicode); +extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode); extern int __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf); extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen); extern const char *__wt_json_tokname(int toktype); @@ -484,6 +483,31 @@ extern int __wt_meta_track_destroy(WT_SESSION_IMPL *session); extern int __wt_turtle_init(WT_SESSION_IMPL *session); extern int __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep); extern int __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value); +extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path); +extern int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path); +extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name); +extern int __wt_rename_and_sync_directory( WT_SESSION_IMPL *session, const char *from, const char *to); +extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to); +extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp); +extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp); +extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); +extern int __wt_realloc_noclear(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); +extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); +extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp); +extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg); +extern int __wt_errno(void); +extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen); +extern int __wt_ext_map_windows_error( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, uint32_t windows_error); +extern bool __wt_handle_is_open(WT_SESSION_IMPL *session, const char *name); +extern int __wt_open(WT_SESSION_IMPL *session, const char *name, WT_OPEN_FILE_TYPE file_type, u_int flags, WT_FH **fhp); +extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp); +extern int __wt_close_connection_close(WT_SESSION_IMPL *session); +extern int __wt_os_inmemory(WT_SESSION_IMPL *session); +extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, uint32_t open_flags, uint32_t flags, WT_FSTREAM **fstrp); +extern int __wt_os_stdio(WT_SESSION_IMPL *session); +extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr); +extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base); extern int __wt_ext_struct_pack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, void *buffer, size_t size, const char *fmt, ...); extern int __wt_ext_struct_size(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t *sizep, const char *fmt, ...); extern int __wt_ext_struct_unpack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const void *buffer, size_t size, const char *fmt, ...); @@ -598,7 +622,14 @@ extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_ extern int __wt_ext_msg_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); extern const char *__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error); extern int __wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v); -extern void __wt_assert(WT_SESSION_IMPL *session, int error, const char *file_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))); +extern void +__wt_assert(WT_SESSION_IMPL *session, + int error, const char *file_name, int line_number, const char *fmt, ...) + WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 5, 6))) +#ifdef HAVE_DIAGNOSTIC + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) +#endif +; extern int __wt_panic(WT_SESSION_IMPL *session); extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name); extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri); @@ -647,6 +678,8 @@ extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state); extern int __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size); extern int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); +extern const char *__wt_buf_set_printable( WT_SESSION_IMPL *session, const void *p, size_t size, WT_ITEM *buf); +extern const char *__wt_buf_set_size( WT_SESSION_IMPL *session, uint64_t size, bool exact, WT_ITEM *buf); extern int __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp #ifdef HAVE_DIAGNOSTIC @@ -676,7 +709,7 @@ extern void __wt_stat_join_clear_all(WT_JOIN_STATS **stats); extern void __wt_stat_join_aggregate( WT_JOIN_STATS **from, WT_JOIN_STATS *to); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); extern int __wt_txn_get_snapshot(WT_SESSION_IMPL *session); -extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, bool force); +extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags); extern int __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]); extern void __wt_txn_release(WT_SESSION_IMPL *session); extern int __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]); @@ -711,64 +744,3 @@ extern int __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM extern int __wt_txn_named_snapshot_config(WT_SESSION_IMPL *session, const char *cfg[], bool *has_create, bool *has_drops); extern int __wt_txn_named_snapshot_destroy(WT_SESSION_IMPL *session); extern int __wt_txn_recover(WT_SESSION_IMPL *session); -extern bool __wt_absolute_path(const char *path); -extern bool __wt_handle_is_open(WT_SESSION_IMPL *session, const char *name); -extern bool __wt_has_priv(void); -extern const char *__wt_path_separator(void); -extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen); -extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp); -extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp); -extern int __wt_close_connection_close(WT_SESSION_IMPL *session); -extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, bool is_signalled, WT_CONDVAR **condp); -extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); -extern int __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); -extern int __wt_cond_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled); -extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to); -extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh); -extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp); -extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret); -extern int __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); -extern int __wt_errno(void); -extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path); -extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, uint32_t open_flags, uint32_t flags, WT_FSTREAM **fstrp); -extern int __wt_get_vm_pagesize(void); -extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp); -extern int __wt_getlasterror(void); -extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr); -extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp); -extern int __wt_map_error_rdonly(int error); -extern int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path); -extern int __wt_once(void (*init_routine)(void)); -extern int __wt_open(WT_SESSION_IMPL *session, const char *name, WT_OPEN_FILE_TYPE file_type, u_int flags, WT_FH **fhp); -extern int __wt_os_inmemory(WT_SESSION_IMPL *session); -extern int __wt_os_posix(WT_SESSION_IMPL *session); -extern int __wt_os_stdio(WT_SESSION_IMPL *session); -extern int __wt_os_win(WT_SESSION_IMPL *session); -extern int __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp); -extern int __wt_posix_directory_list_free(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, char **dirlist, uint32_t count); -extern int __wt_posix_file_fallocate(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset, wt_off_t len); -extern int __wt_posix_map(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep); -extern int __wt_posix_map_discard(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *map, size_t length, void *mapped_cookie); -extern int __wt_posix_map_preload(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, const void *map, size_t length, void *mapped_cookie); -extern int __wt_posix_unmap(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_region, size_t len, void *mapped_cookie); -extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); -extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); -extern int __wt_realloc_noclear(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); -extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name); -extern int __wt_rename_and_sync_directory( WT_SESSION_IMPL *session, const char *from, const char *to); -extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp); -extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg); -extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid); -extern int __wt_win_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp); -extern int __wt_win_directory_list_free(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, char **dirlist, uint32_t count); -extern int __wt_win_fs_size(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, wt_off_t *sizep); -extern int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep); -extern int __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_region, size_t length, void *mapped_cookie); -extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base); -extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); -extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg); -extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); -extern void __wt_stream_set_line_buffer(FILE *fp); -extern void __wt_stream_set_no_buffer(FILE *fp); -extern void __wt_thread_id(char *buf, size_t buflen); -extern void __wt_yield(void); diff --git a/src/include/extern_posix.h b/src/include/extern_posix.h new file mode 100644 index 00000000000..6fde537f36b --- /dev/null +++ b/src/include/extern_posix.h @@ -0,0 +1,31 @@ +/* DO NOT EDIT: automatically built by dist/s_prototypes. */ + +extern int __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp); +extern int __wt_posix_directory_list_free(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, char **dirlist, uint32_t count); +extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp); +extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret); +extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh); +extern int __wt_posix_file_fallocate(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset, wt_off_t len); +extern int __wt_os_posix(WT_SESSION_IMPL *session); +extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp); +extern int __wt_posix_map(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep); +extern int __wt_posix_map_preload(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, const void *map, size_t length, void *mapped_cookie); +extern int __wt_posix_map_discard(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *map, size_t length, void *mapped_cookie); +extern int __wt_posix_unmap(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_region, size_t len, void *mapped_cookie); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, bool is_signalled, WT_CONDVAR **condp); +extern int __wt_cond_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled); +extern int __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); +extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); +extern int __wt_once(void (*init_routine)(void)); +extern int __wt_get_vm_pagesize(void); +extern bool __wt_absolute_path(const char *path); +extern const char *__wt_path_separator(void); +extern bool __wt_has_priv(void); +extern void __wt_stream_set_line_buffer(FILE *fp); +extern void __wt_stream_set_no_buffer(FILE *fp); +extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); +extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg); +extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid); +extern void __wt_thread_id(char *buf, size_t buflen); +extern int __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); +extern void __wt_yield(void); diff --git a/src/include/extern_win.h b/src/include/extern_win.h new file mode 100644 index 00000000000..c5c2624db2c --- /dev/null +++ b/src/include/extern_win.h @@ -0,0 +1,32 @@ +/* DO NOT EDIT: automatically built by dist/s_prototypes. */ + +extern int __wt_win_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp); +extern int __wt_win_directory_list_free(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, char **dirlist, uint32_t count); +extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp); +extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret); +extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh); +extern int __wt_win_fs_size(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, wt_off_t *sizep); +extern int __wt_os_win(WT_SESSION_IMPL *session); +extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp); +extern int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep); +extern int __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_region, size_t length, void *mapped_cookie); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, bool is_signalled, WT_CONDVAR **condp); +extern int __wt_cond_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled); +extern int __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); +extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); +extern int __wt_once(void (*init_routine)(void)); +extern int __wt_get_vm_pagesize(void); +extern bool __wt_absolute_path(const char *path); +extern const char *__wt_path_separator(void); +extern bool __wt_has_priv(void); +extern void __wt_stream_set_line_buffer(FILE *fp); +extern void __wt_stream_set_no_buffer(FILE *fp); +extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); +extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg); +extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid); +extern void __wt_thread_id(char *buf, size_t buflen); +extern int __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); +extern DWORD __wt_getlasterror(void); +extern int __wt_map_windows_error(DWORD windows_error); +extern const char *__wt_formatmessage(WT_SESSION_IMPL *session, DWORD windows_error); +extern void __wt_yield(void); diff --git a/src/include/flags.h b/src/include/flags.h index da7aee7b059..f134af69d29 100644 --- a/src/include/flags.h +++ b/src/include/flags.h @@ -13,13 +13,14 @@ #define WT_CONN_LSM_MERGE 0x00000100 #define WT_CONN_PANIC 0x00000200 #define WT_CONN_READONLY 0x00000400 -#define WT_CONN_SERVER_ASYNC 0x00000800 -#define WT_CONN_SERVER_CHECKPOINT 0x00001000 -#define WT_CONN_SERVER_LSM 0x00002000 -#define WT_CONN_SERVER_RUN 0x00004000 -#define WT_CONN_SERVER_STATISTICS 0x00008000 -#define WT_CONN_SERVER_SWEEP 0x00010000 -#define WT_CONN_WAS_BACKUP 0x00020000 +#define WT_CONN_RECOVERING 0x00000800 +#define WT_CONN_SERVER_ASYNC 0x00001000 +#define WT_CONN_SERVER_CHECKPOINT 0x00002000 +#define WT_CONN_SERVER_LSM 0x00004000 +#define WT_CONN_SERVER_RUN 0x00008000 +#define WT_CONN_SERVER_STATISTICS 0x00010000 +#define WT_CONN_SERVER_SWEEP 0x00020000 +#define WT_CONN_WAS_BACKUP 0x00040000 #define WT_EVICTING 0x00000001 #define WT_EVICT_IN_MEMORY 0x00000002 #define WT_EVICT_LOOKASIDE 0x00000004 @@ -47,11 +48,11 @@ #define WT_READ_TRUNCATE 0x00000800 #define WT_READ_WONT_NEED 0x00001000 #define WT_SESSION_CAN_WAIT 0x00000001 -#define WT_SESSION_CLEAR_EVICT_WALK 0x00000002 -#define WT_SESSION_INTERNAL 0x00000004 -#define WT_SESSION_LOCKED_CHECKPOINT 0x00000008 -#define WT_SESSION_LOCKED_HANDLE_LIST 0x00000010 -#define WT_SESSION_LOCKED_METADATA 0x00000020 +#define WT_SESSION_INTERNAL 0x00000002 +#define WT_SESSION_LOCKED_CHECKPOINT 0x00000004 +#define WT_SESSION_LOCKED_HANDLE_LIST 0x00000008 +#define WT_SESSION_LOCKED_METADATA 0x00000010 +#define WT_SESSION_LOCKED_PASS 0x00000020 #define WT_SESSION_LOCKED_SCHEMA 0x00000040 #define WT_SESSION_LOCKED_SLOT 0x00000080 #define WT_SESSION_LOCKED_TABLE 0x00000100 @@ -71,6 +72,8 @@ #define WT_TXN_LOG_CKPT_START 0x00000004 #define WT_TXN_LOG_CKPT_STOP 0x00000008 #define WT_TXN_LOG_CKPT_SYNC 0x00000010 +#define WT_TXN_OLDEST_STRICT 0x00000001 +#define WT_TXN_OLDEST_WAIT 0x00000002 #define WT_VERB_API 0x00000001 #define WT_VERB_BLOCK 0x00000002 #define WT_VERB_CHECKPOINT 0x00000004 diff --git a/src/include/log.h b/src/include/log.h index 7655cfbb3e9..870c046252c 100644 --- a/src/include/log.h +++ b/src/include/log.h @@ -256,9 +256,8 @@ struct __wt_log { #ifdef HAVE_DIAGNOSTIC uint64_t write_calls; /* Calls to log_write */ #endif - -#define WT_LOG_NOT_VERIFIED 0x1 /* Log just started */ - uint32_t flags; +#define WT_LOG_OPENED 0x01 /* Log subsystem successfully open */ + uint32_t flags; }; struct __wt_log_record { diff --git a/src/include/misc.h b/src/include/misc.h index 4c7c9572905..1121b7dfa75 100644 --- a/src/include/misc.h +++ b/src/include/misc.h @@ -12,6 +12,8 @@ */ #define WT_UNUSED(var) (void)(var) +#define WT_DIVIDER "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + /* Basic constants. */ #define WT_THOUSAND (1000) #define WT_MILLION (1000000) @@ -31,12 +33,12 @@ */ #define WT_STORE_SIZE(s) ((uint32_t)(s)) #define WT_PTRDIFF(end, begin) \ - ((size_t)((uint8_t *)(end) - (uint8_t *)(begin))) + ((size_t)((const uint8_t *)(end) - (const uint8_t *)(begin))) #define WT_PTRDIFF32(end, begin) \ WT_STORE_SIZE(WT_PTRDIFF((end), (begin))) #define WT_BLOCK_FITS(p, len, begin, maxlen) \ - ((uint8_t *)(p) >= (uint8_t *)(begin) && \ - ((uint8_t *)(p) + (len) <= (uint8_t *)(begin) + (maxlen))) + ((const uint8_t *)(p) >= (const uint8_t *)(begin) && \ + ((const uint8_t *)(p) + (len) <= (const uint8_t *)(begin) + (maxlen))) #define WT_PTR_IN_RANGE(p, begin, maxlen) \ WT_BLOCK_FITS((p), 1, (begin), (maxlen)) diff --git a/src/include/os.h b/src/include/os.h index dd9b96f73a8..7a8e47ed81f 100644 --- a/src/include/os.h +++ b/src/include/os.h @@ -6,28 +6,32 @@ * See the file LICENSE for redistribution information. */ +#define WT_SYSCALL(call, ret) do { \ + /* \ + * A call returning 0 indicates success; any call where \ + * 0 is not the only successful return must provide an \ + * expression evaluating to 0 in all successful cases. \ + */ \ + if (((ret) = (call)) == 0) \ + break; \ + /* \ + * The call's error was either returned by the call or \ + * is in errno, and there are cases where it depends on \ + * the software release as to which it is (for example, \ + * posix_fadvise on FreeBSD and OS X). Failing calls \ + * must either return a non-zero error value, or -1 if \ + * the error value is in errno. (The WiredTiger errno \ + * function returns WT_ERROR if errno is 0, which isn't \ + * ideal but won't discard the failure.) \ + */ \ + if ((ret) == -1) \ + (ret) = __wt_errno(); \ +} while (0) + #define WT_SYSCALL_RETRY(call, ret) do { \ int __retry; \ for (__retry = 0; __retry < 10; ++__retry) { \ - /* \ - * A call returning 0 indicates success; any call where \ - * 0 is not the only successful return must provide an \ - * expression evaluating to 0 in all successful cases. \ - */ \ - if (((ret) = (call)) == 0) \ - break; \ - /* \ - * The call's error was either returned by the call or \ - * is in errno, and there are cases where it depends on \ - * the software release as to which it is (for example, \ - * posix_fadvise on FreeBSD and OS X). Failing calls \ - * must either return a non-zero error value, or -1 if \ - * the error value is in errno. (The WiredTiger errno \ - * function returns WT_ERROR if errno is 0, which isn't \ - * ideal but won't discard the failure.) \ - */ \ - if ((ret) == -1) \ - (ret) = __wt_errno(); \ + WT_SYSCALL(call, ret); \ switch (ret) { \ case EAGAIN: \ case EBUSY: \ @@ -132,7 +136,6 @@ struct __wt_file_handle_inmem { TAILQ_ENTRY(__wt_file_handle_inmem) q; /* internal queue, hash queue */ TAILQ_ENTRY(__wt_file_handle_inmem) hashq; - size_t off; /* Read/write offset */ WT_ITEM buf; /* Data */ u_int ref; /* Reference count */ }; diff --git a/src/include/os_fhandle.i b/src/include/os_fhandle.i index cf790d6bc4d..313bf8eca3f 100644 --- a/src/include/os_fhandle.i +++ b/src/include/os_fhandle.i @@ -13,6 +13,7 @@ static inline int __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh, bool block) { + WT_DECL_RET; WT_FILE_HANDLE *handle; WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); @@ -21,12 +22,20 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh, bool block) session, WT_VERB_HANDLEOPS, "%s: handle-sync", fh->handle->name)); handle = fh->handle; + /* + * There is no way to check when the non-blocking sync-file-range is + * complete, but we track the time taken in the call for completeness. + */ + WT_STAT_FAST_CONN_INCR_ATOMIC(session, fsync_active); + WT_STAT_FAST_CONN_INCR(session, fsync_io); if (block) - return (handle->fh_sync == NULL ? 0 : + ret = (handle->fh_sync == NULL ? 0 : handle->fh_sync(handle, (WT_SESSION *)session)); else - return (handle->fh_sync_nowait == NULL ? 0 : + ret = (handle->fh_sync_nowait == NULL ? 0 : handle->fh_sync_nowait(handle, (WT_SESSION *)session)); + WT_STAT_FAST_CONN_DECR_ATOMIC(session, fsync_active); + return (ret); } /* @@ -92,14 +101,20 @@ static inline int __wt_read( WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, size_t len, void *buf) { + WT_DECL_RET; + WT_RET(__wt_verbose(session, WT_VERB_HANDLEOPS, "%s: handle-read: %" WT_SIZET_FMT " at %" PRIuMAX, fh->handle->name, len, (uintmax_t)offset)); + WT_STAT_FAST_CONN_INCR_ATOMIC(session, read_active); WT_STAT_FAST_CONN_INCR(session, read_io); - return (fh->handle->fh_read( - fh->handle, (WT_SESSION *)session, offset, len, buf)); + ret = fh->handle->fh_read( + fh->handle, (WT_SESSION *)session, offset, len, buf); + + WT_STAT_FAST_CONN_DECR_ATOMIC(session, read_active); + return (ret); } /* @@ -140,6 +155,8 @@ static inline int __wt_write(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, size_t len, const void *buf) { + WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, strlen(WT_SINGLETHREAD))); @@ -148,8 +165,12 @@ __wt_write(WT_SESSION_IMPL *session, "%s: handle-write: %" WT_SIZET_FMT " at %" PRIuMAX, fh->handle->name, len, (uintmax_t)offset)); + WT_STAT_FAST_CONN_INCR_ATOMIC(session, write_active); WT_STAT_FAST_CONN_INCR(session, write_io); - return (fh->handle->fh_write( - fh->handle, (WT_SESSION *)session, offset, len, buf)); + ret = fh->handle->fh_write( + fh->handle, (WT_SESSION *)session, offset, len, buf); + + WT_STAT_FAST_CONN_DECR_ATOMIC(session, write_active); + return (ret); } diff --git a/src/include/packing.i b/src/include/packing.i index 9d5971ed99f..d662c60d221 100644 --- a/src/include/packing.i +++ b/src/include/packing.i @@ -138,7 +138,7 @@ __pack_next(WT_PACK *pack, WT_PACK_VALUE *pv) next: if (pack->cur == pack->end) return (WT_NOTFOUND); - if (isdigit(*pack->cur)) { + if (__wt_isdigit((u_char)*pack->cur)) { pv->havesize = 1; pv->size = WT_STORE_SIZE(strtoul(pack->cur, &endsize, 10)); pack->cur = endsize; @@ -540,7 +540,7 @@ __unpack_read(WT_SESSION_IMPL *session, break; case 'R': WT_SIZE_CHECK_UNPACK(sizeof(uint64_t), maxlen); - pv->u.u = *(uint64_t *)*pp; + pv->u.u = *(const uint64_t *)*pp; *pp += sizeof(uint64_t); break; default: diff --git a/src/include/serial.i b/src/include/serial.i index c0cd9c85ee9..67edc1c9ce1 100644 --- a/src/include/serial.i +++ b/src/include/serial.i @@ -306,7 +306,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, if ((txn = page->modify->obsolete_check_txn) != WT_TXN_NONE) { if (!__wt_txn_visible_all(session, txn)) { /* Try to move the oldest ID forward and re-check. */ - WT_RET(__wt_txn_update_oldest(session, false)); + WT_RET(__wt_txn_update_oldest(session, 0)); if (!__wt_txn_visible_all(session, txn)) return (0); diff --git a/src/include/stat.h b/src/include/stat.h index a71e0fa208e..57126af8aa4 100644 --- a/src/include/stat.h +++ b/src/include/stat.h @@ -144,10 +144,16 @@ __wt_stats_clear(void *stats_arg, int slot) #define WT_STAT_DECRV(session, stats, fld, value) \ (stats)[WT_STATS_SLOT_ID(session)]->fld -= (int64_t)(value) +#define WT_STAT_DECRV_ATOMIC(session, stats, fld, value) \ + __wt_atomic_addi64( \ + &(stats)[WT_STATS_SLOT_ID(session)]->fld, (int64_t)(value)) #define WT_STAT_DECR(session, stats, fld) \ WT_STAT_DECRV(session, stats, fld, 1) #define WT_STAT_INCRV(session, stats, fld, value) \ (stats)[WT_STATS_SLOT_ID(session)]->fld += (int64_t)(value) +#define WT_STAT_INCRV_ATOMIC(session, stats, fld, value) \ + __wt_atomic_subi64( \ + &(stats)[WT_STATS_SLOT_ID(session)]->fld, (int64_t)(value)) #define WT_STAT_INCR(session, stats, fld) \ WT_STAT_INCRV(session, stats, fld, 1) #define WT_STAT_SET(session, stats, fld, value) do { \ @@ -164,12 +170,20 @@ __wt_stats_clear(void *stats_arg, int slot) } while (0) #define WT_STAT_FAST_DECR(session, stats, fld) \ WT_STAT_FAST_DECRV(session, stats, fld, 1) +#define WT_STAT_FAST_DECRV_ATOMIC(session, stats, fld, value) do { \ + if (FLD_ISSET(S2C(session)->stat_flags, WT_CONN_STAT_FAST)) \ + WT_STAT_DECRV_ATOMIC(session, stats, fld, value); \ +} while (0) #define WT_STAT_FAST_INCRV(session, stats, fld, value) do { \ if (FLD_ISSET(S2C(session)->stat_flags, WT_CONN_STAT_FAST)) \ WT_STAT_INCRV(session, stats, fld, value); \ } while (0) #define WT_STAT_FAST_INCR(session, stats, fld) \ WT_STAT_FAST_INCRV(session, stats, fld, 1) +#define WT_STAT_FAST_INCRV_ATOMIC(session, stats, fld, value) do { \ + if (FLD_ISSET(S2C(session)->stat_flags, WT_CONN_STAT_FAST)) \ + WT_STAT_INCRV_ATOMIC(session, stats, fld, value); \ +} while (0) #define WT_STAT_FAST_SET(session, stats, fld, value) do { \ if (FLD_ISSET(S2C(session)->stat_flags, WT_CONN_STAT_FAST)) \ WT_STAT_SET(session, stats, fld, value); \ @@ -180,10 +194,14 @@ __wt_stats_clear(void *stats_arg, int slot) */ #define WT_STAT_FAST_CONN_DECR(session, fld) \ WT_STAT_FAST_DECR(session, S2C(session)->stats, fld) +#define WT_STAT_FAST_CONN_DECR_ATOMIC(session, fld) \ + WT_STAT_FAST_DECRV_ATOMIC(session, S2C(session)->stats, fld, 1) #define WT_STAT_FAST_CONN_DECRV(session, fld, value) \ WT_STAT_FAST_DECRV(session, S2C(session)->stats, fld, value) #define WT_STAT_FAST_CONN_INCR(session, fld) \ WT_STAT_FAST_INCR(session, S2C(session)->stats, fld) +#define WT_STAT_FAST_CONN_INCR_ATOMIC(session, fld) \ + WT_STAT_FAST_INCRV_ATOMIC(session, S2C(session)->stats, fld, 1) #define WT_STAT_FAST_CONN_INCRV(session, fld, value) \ WT_STAT_FAST_INCRV(session, S2C(session)->stats, fld, value) #define WT_STAT_FAST_CONN_SET(session, fld, value) \ @@ -274,6 +292,8 @@ struct __wt_connection_stats { int64_t cache_eviction_slow; int64_t cache_eviction_worker_evicting; int64_t cache_eviction_force_fail; + int64_t cache_eviction_walks_active; + int64_t cache_eviction_walks_started; int64_t cache_eviction_hazard; int64_t cache_hazard_checks; int64_t cache_hazard_walks; @@ -288,15 +308,19 @@ struct __wt_connection_stats { int64_t cache_bytes_max; int64_t cache_eviction_maximum_page_size; int64_t cache_eviction_dirty; + int64_t cache_eviction_app_dirty; int64_t cache_eviction_deepen; int64_t cache_write_lookaside; int64_t cache_pages_inuse; int64_t cache_eviction_force; int64_t cache_eviction_force_delete; int64_t cache_eviction_app; + int64_t cache_eviction_pages_queued; + int64_t cache_eviction_pages_queued_oldest; int64_t cache_read; int64_t cache_read_lookaside; int64_t cache_pages_requested; + int64_t cache_eviction_pages_seen; int64_t cache_eviction_fail; int64_t cache_eviction_walk; int64_t cache_write; @@ -317,6 +341,7 @@ struct __wt_connection_stats { int64_t cond_wait; int64_t rwlock_read; int64_t rwlock_write; + int64_t fsync_io; int64_t read_io; int64_t write_io; int64_t cursor_create; @@ -359,7 +384,9 @@ struct __wt_connection_stats { int64_t log_write_lsn; int64_t log_write_lsn_skip; int64_t log_sync; + int64_t log_sync_duration; int64_t log_sync_dir; + int64_t log_sync_dir_duration; int64_t log_writes; int64_t log_slot_consolidated; int64_t log_max_filesize; @@ -381,6 +408,9 @@ struct __wt_connection_stats { int64_t rec_split_stashed_objects; int64_t session_cursor_open; int64_t session_open; + int64_t fsync_active; + int64_t read_active; + int64_t write_active; int64_t page_busy_blocked; int64_t page_forcible_evict_blocked; int64_t page_locked_blocked; @@ -397,6 +427,10 @@ struct __wt_connection_stats { int64_t txn_checkpoint_time_total; int64_t txn_checkpoint; int64_t txn_fail_cache; + int64_t txn_checkpoint_fsync_post; + int64_t txn_checkpoint_fsync_pre; + int64_t txn_checkpoint_fsync_post_duration; + int64_t txn_checkpoint_fsync_pre_duration; int64_t txn_pinned_range; int64_t txn_pinned_checkpoint_range; int64_t txn_pinned_snapshot_range; @@ -517,9 +551,11 @@ struct __wt_dsrc_stats { */ #define WT_JOIN_STATS_BASE 3000 struct __wt_join_stats { - int64_t accesses; - int64_t actual_count; + int64_t main_access; int64_t bloom_false_positive; + int64_t membership_check; + int64_t bloom_insert; + int64_t iterated; }; /* Statistics section: END */ diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 6de92b72051..f578f4e6c08 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1466,15 +1466,16 @@ struct __wt_session { * @config{dump_blocks, Display the contents of on-disk blocks as they * are verified\, using the application's message handler\, intended for * debugging., a boolean flag; default \c false.} + * @config{dump_layout, Display the layout of the files as they are + * verified\, using the application's message handler\, intended for + * debugging; requires optional support from the block manager., a + * boolean flag; default \c false.} * @config{dump_offsets, Display the contents of specific on-disk * blocks\, using the application's message handler\, intended for * debugging., a list of strings; default empty.} * @config{dump_pages, Display the contents of in-memory pages as they * are verified\, using the application's message handler\, intended for * debugging., a boolean flag; default \c false.} - * @config{dump_shape, Display the shape of the tree after - * verification\, using the application's message handler\, intended for - * debugging., a boolean flag; default \c false.} * @config{strict, Treat any verification problem as an error; by * default\, verify will warn\, but not fail\, in the case of errors * that won't affect future behavior (for example\, a leaked block)., a @@ -1848,7 +1849,7 @@ struct __wt_connection { * @config{ path, the path to a directory into * which the log files are written. If the value is not an absolute * path name\, the files are created relative to the database home., a - * string; default empty.} + * string; default \c ".".} * @config{ prealloc, * pre-allocate log files., a boolean flag; default \c true.} * @config{ recover, run recovery or error if @@ -2344,7 +2345,7 @@ struct __wt_connection { * integer between 100KB and 2GB; default \c 100MB.} * @config{ path, the path to a directory into which the * log files are written. If the value is not an absolute path name\, the files - * are created relative to the database home., a string; default empty.} + * are created relative to the database home., a string; default \c ".".} * @config{ prealloc, pre-allocate log files., a boolean * flag; default \c true.} * @config{ recover, run recovery @@ -3050,10 +3051,6 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); * transaction. */ #define WT_CACHE_FULL -31807 -/*! @cond internal */ -/*! Permission denied (internal). */ -#define WT_PERM_DENIED -31808 -/*! @endcond */ /* * Error return section: END * DO NOT EDIT: automatically built by dist/api_err.py. @@ -4298,265 +4295,301 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1045 /*! cache: failed eviction of pages that exceeded the in-memory maximum */ #define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1046 +/*! cache: files with active eviction walks */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1047 +/*! cache: files with new eviction walks started */ +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1048 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1047 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1049 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1048 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1050 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1049 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1051 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1050 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1052 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1051 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1053 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1052 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1054 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1053 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1055 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1054 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1056 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1055 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1057 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1056 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1058 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1057 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1059 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1058 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1060 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1059 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1061 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1060 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1062 +/*! cache: modified pages evicted by application threads */ +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1063 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1061 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1064 /*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1062 +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1065 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1063 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1066 /*! cache: pages evicted because they exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1064 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1067 /*! cache: pages evicted because they had chains of deleted items */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1065 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1068 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1066 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1069 +/*! cache: pages queued for eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1070 +/*! cache: pages queued for urgent eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1071 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1067 +#define WT_STAT_CONN_CACHE_READ 1072 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1068 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1073 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1069 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1074 +/*! cache: pages seen by eviction walk */ +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1075 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1070 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1076 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1071 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1077 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1072 +#define WT_STAT_CONN_CACHE_WRITE 1078 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1073 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1079 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1074 +#define WT_STAT_CONN_CACHE_OVERHEAD 1080 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1075 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1081 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1076 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1082 /*! cache: tracked bytes belonging to overflow pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_OVERFLOW 1077 +#define WT_STAT_CONN_CACHE_BYTES_OVERFLOW 1083 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1078 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1084 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1079 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1085 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1080 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1086 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1081 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1087 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1082 +#define WT_STAT_CONN_COND_AUTO_WAIT 1088 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1083 +#define WT_STAT_CONN_FILE_OPEN 1089 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1084 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1090 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1085 +#define WT_STAT_CONN_MEMORY_FREE 1091 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1086 +#define WT_STAT_CONN_MEMORY_GROW 1092 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1087 +#define WT_STAT_CONN_COND_WAIT 1093 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1088 +#define WT_STAT_CONN_RWLOCK_READ 1094 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1089 +#define WT_STAT_CONN_RWLOCK_WRITE 1095 +/*! connection: total fsync I/Os */ +#define WT_STAT_CONN_FSYNC_IO 1096 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1090 +#define WT_STAT_CONN_READ_IO 1097 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1091 +#define WT_STAT_CONN_WRITE_IO 1098 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1092 +#define WT_STAT_CONN_CURSOR_CREATE 1099 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1093 +#define WT_STAT_CONN_CURSOR_INSERT 1100 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1094 +#define WT_STAT_CONN_CURSOR_NEXT 1101 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1095 +#define WT_STAT_CONN_CURSOR_PREV 1102 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1096 +#define WT_STAT_CONN_CURSOR_REMOVE 1103 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1097 +#define WT_STAT_CONN_CURSOR_RESET 1104 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1098 +#define WT_STAT_CONN_CURSOR_RESTART 1105 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1099 +#define WT_STAT_CONN_CURSOR_SEARCH 1106 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1100 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1107 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1101 +#define WT_STAT_CONN_CURSOR_UPDATE 1108 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1102 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1109 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1103 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1110 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1104 +#define WT_STAT_CONN_DH_SWEEP_REF 1111 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1105 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1112 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1106 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1113 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1107 +#define WT_STAT_CONN_DH_SWEEP_TOD 1114 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1108 +#define WT_STAT_CONN_DH_SWEEPS 1115 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1109 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1116 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1110 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1117 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1111 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1118 /*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1112 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1119 /*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1113 +#define WT_STAT_CONN_LOG_SLOT_RACES 1120 /*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1114 +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1121 /*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1115 +#define WT_STAT_CONN_LOG_SLOT_JOINS 1122 /*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1116 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1123 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1117 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1124 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1118 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1125 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1119 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1126 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1120 +#define WT_STAT_CONN_LOG_FLUSH 1127 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1121 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1128 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1122 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1129 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1123 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1130 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1124 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1131 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1125 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1132 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1126 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1133 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1127 +#define WT_STAT_CONN_LOG_SCANS 1134 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1128 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1135 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1129 +#define WT_STAT_CONN_LOG_WRITE_LSN 1136 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1130 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1137 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1131 +#define WT_STAT_CONN_LOG_SYNC 1138 +/*! log: log sync time duration (usecs) */ +#define WT_STAT_CONN_LOG_SYNC_DURATION 1139 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1132 +#define WT_STAT_CONN_LOG_SYNC_DIR 1140 +/*! log: log sync_dir time duration (usecs) */ +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1141 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1133 +#define WT_STAT_CONN_LOG_WRITES 1142 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1134 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1143 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1135 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1144 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1136 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1145 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1137 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1146 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1138 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1147 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1139 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1148 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1140 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1149 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1141 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1150 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1142 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1151 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1143 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1152 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1144 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1153 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1145 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1154 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1146 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1155 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1147 +#define WT_STAT_CONN_REC_PAGES 1156 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1148 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1157 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1149 +#define WT_STAT_CONN_REC_PAGE_DELETE 1158 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1150 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1159 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1151 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1160 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1152 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1161 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1153 +#define WT_STAT_CONN_SESSION_OPEN 1162 +/*! thread-state: active filesystem fsync calls */ +#define WT_STAT_CONN_FSYNC_ACTIVE 1163 +/*! thread-state: active filesystem read calls */ +#define WT_STAT_CONN_READ_ACTIVE 1164 +/*! thread-state: active filesystem write calls */ +#define WT_STAT_CONN_WRITE_ACTIVE 1165 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1154 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1166 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1155 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1167 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1156 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1168 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1157 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1169 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1158 +#define WT_STAT_CONN_PAGE_SLEEP 1170 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1159 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1171 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1160 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1172 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1161 +#define WT_STAT_CONN_TXN_BEGIN 1173 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1162 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1174 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1163 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1175 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1164 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1176 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1165 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1177 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1166 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1178 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1167 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1179 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1168 +#define WT_STAT_CONN_TXN_CHECKPOINT 1180 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1169 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1181 +/*! transaction: transaction fsync calls for checkpoint after allocating + * the transaction ID */ +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1182 +/*! transaction: transaction fsync calls for checkpoint before allocating + * the transaction ID */ +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_PRE 1183 +/*! transaction: transaction fsync duration for checkpoint after + * allocating the transaction ID (usecs) */ +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1184 +/*! transaction: transaction fsync duration for checkpoint before + * allocating the transaction ID (usecs) */ +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_PRE_DURATION 1185 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1170 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1186 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1171 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1187 /*! transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1172 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1188 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1173 +#define WT_STAT_CONN_TXN_SYNC 1189 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1174 +#define WT_STAT_CONN_TXN_COMMIT 1190 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1175 +#define WT_STAT_CONN_TXN_ROLLBACK 1191 /*! * @} @@ -4773,12 +4806,16 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * @anchor statistics_join * @{ */ -/*! : accesses */ -#define WT_STAT_JOIN_ACCESSES 3000 -/*! : actual count of items */ -#define WT_STAT_JOIN_ACTUAL_COUNT 3001 +/*! : accesses to the main table */ +#define WT_STAT_JOIN_MAIN_ACCESS 3000 /*! : bloom filter false positives */ -#define WT_STAT_JOIN_BLOOM_FALSE_POSITIVE 3002 +#define WT_STAT_JOIN_BLOOM_FALSE_POSITIVE 3001 +/*! : checks that conditions of membership are satisfied */ +#define WT_STAT_JOIN_MEMBERSHIP_CHECK 3002 +/*! : items inserted into a bloom filter */ +#define WT_STAT_JOIN_BLOOM_INSERT 3003 +/*! : items iterated */ +#define WT_STAT_JOIN_ITERATED 3004 /*! @} */ /* * Statistics section: END diff --git a/src/include/wiredtiger_ext.h b/src/include/wiredtiger_ext.h index 7d97d97dcf5..3d65cd1fc24 100644 --- a/src/include/wiredtiger_ext.h +++ b/src/include/wiredtiger_ext.h @@ -131,6 +131,19 @@ struct __wt_extension_api { WT_EXTENSION_API *, WT_SESSION *session, int error); /*! + * Map a Windows system error code to a POSIX 1003.1/ANSI C error. + * + * @param wt_api the extension handle + * @param session the session handle (or NULL if none available) + * @param windows_error a Windows system error code + * @returns a string representation of the error + * + * @snippet ex_data_source.c WT_EXTENSION_API map_windows_error + */ + int (*map_windows_error)(WT_EXTENSION_API *wt_api, + WT_SESSION *session, uint32_t windows_error); + + /*! * Allocate short-term use scratch memory. * * @param wt_api the extension handle diff --git a/src/include/wt_internal.h b/src/include/wt_internal.h index 0c8abf36cfe..c5337967f22 100644 --- a/src/include/wt_internal.h +++ b/src/include/wt_internal.h @@ -28,7 +28,6 @@ extern "C" { #include <sys/time.h> #include <sys/uio.h> #endif -#include <ctype.h> #ifndef _WIN32 #include <dlfcn.h> #endif @@ -357,8 +356,14 @@ union __wt_rand_state; #include "connection.h" #include "extern.h" +#ifdef _WIN32 +#include "extern_win.h" +#else +#include "extern_posix.h" +#endif #include "verify_build.h" +#include "ctype.i" /* required by packing.i */ #include "intpack.i" /* required by cell.i, packing.i */ #include "buf.i" /* required by cell.i */ diff --git a/src/log/log.c b/src/log/log.c index 56e9f65f914..bf83c280d8d 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -95,9 +95,11 @@ __wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn) int __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) { + struct timespec fsync_start, fsync_stop; WT_DECL_RET; WT_FH *log_fh; WT_LOG *log; + uint64_t fsync_duration_usecs; log = S2C(session)->log; @@ -124,9 +126,14 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) "log_force_sync: sync directory %s to LSN %" PRIu32 "/%" PRIu32, log->log_dir_fh->name, min_lsn->l.file, min_lsn->l.offset)); + WT_ERR(__wt_epoch(session, &fsync_start)); WT_ERR(__wt_fsync(session, log->log_dir_fh, true)); + WT_ERR(__wt_epoch(session, &fsync_stop)); + fsync_duration_usecs = WT_TIMEDIFF_US(fsync_stop, fsync_start); log->sync_dir_lsn = *min_lsn; WT_STAT_FAST_CONN_INCR(session, log_sync_dir); + WT_STAT_FAST_CONN_INCRV(session, + log_sync_dir_duration, fsync_duration_usecs); } /* * Sync the log file if needed. @@ -143,9 +150,14 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) WT_ERR(__wt_verbose(session, WT_VERB_LOG, "log_force_sync: sync %s to LSN %" PRIu32 "/%" PRIu32, log_fh->name, min_lsn->l.file, min_lsn->l.offset)); + WT_ERR(__wt_epoch(session, &fsync_start)); WT_ERR(__wt_fsync(session, log_fh, true)); + WT_ERR(__wt_epoch(session, &fsync_stop)); + fsync_duration_usecs = WT_TIMEDIFF_US(fsync_stop, fsync_start); log->sync_lsn = *min_lsn; WT_STAT_FAST_CONN_INCR(session, log_sync); + WT_STAT_FAST_CONN_INCRV(session, + log_sync_duration, fsync_duration_usecs); WT_ERR(__wt_close(session, &log_fh)); WT_ERR(__wt_cond_signal(session, log->log_sync_cond)); } @@ -447,12 +459,12 @@ __log_prealloc(WT_SESSION_IMPL *session, WT_FH *fh) * writes happening concurrently, so there are no locking issues. */ if ((ret = __wt_fallocate( - session, fh, WT_LOG_FIRST_RECORD, conn->log_file_max)) == 0) + session, fh, WT_LOG_FIRST_RECORD, + conn->log_file_max - WT_LOG_FIRST_RECORD)) == 0) return (0); WT_RET_ERROR_OK(ret, ENOTSUP); - return (__wt_ftruncate( - session, fh, WT_LOG_FIRST_RECORD + conn->log_file_max)); + return (__wt_ftruncate(session, fh, conn->log_file_max)); } /* @@ -778,6 +790,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_FH *log_fh; WT_LOG *log; WT_LSN end_lsn; int yield_cnt; @@ -850,8 +863,15 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) WT_RET(__wt_log_allocfile( session, log->fileid, WT_LOG_FILENAME)); } + /* + * Since the file system clears the output file handle pointer before + * searching the handle list and filling in the new file handle, + * we must pass in a local file handle. Otherwise there is a wide + * window where another thread could see a NULL log file handle. + */ WT_RET(__log_openfile(session, - false, &log->log_fh, WT_LOG_FILENAME, log->fileid)); + false, &log_fh, WT_LOG_FILENAME, log->fileid)); + WT_PUBLISH(log->log_fh, log_fh); /* * We need to setup the LSNs. Set the end LSN and alloc LSN to * the end of the header. @@ -1029,7 +1049,6 @@ __wt_log_allocfile( */ WT_ERR(__log_openfile(session, true, &log_fh, WT_LOG_TMPNAME, tmp_id)); WT_ERR(__log_file_header(session, log_fh, NULL, true)); - WT_ERR(__wt_ftruncate(session, log_fh, WT_LOG_FIRST_RECORD)); WT_ERR(__log_prealloc(session, log_fh)); WT_ERR(__wt_fsync(session, log_fh, true)); WT_ERR(__wt_close(session, &log_fh)); @@ -1090,7 +1109,6 @@ __wt_log_open(WT_SESSION_IMPL *session) logcount = 0; lastlog = 0; firstlog = UINT32_MAX; - F_SET(log, WT_LOG_NOT_VERIFIED); /* * Open up a file handle to the log directory if we haven't. @@ -1166,6 +1184,8 @@ __wt_log_open(WT_SESSION_IMPL *session) } err: WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); + if (ret == 0) + F_SET(log, WT_LOG_OPENED); return (ret); } @@ -1205,6 +1225,7 @@ __wt_log_close(WT_SESSION_IMPL *session) WT_RET(__wt_close(session, &log->log_dir_fh)); log->log_dir_fh = NULL; } + F_CLR(log, WT_LOG_OPENED); return (0); } @@ -1274,11 +1295,13 @@ err: __wt_free(session, buf); int __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) { + struct timespec fsync_start, fsync_stop; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; WT_LSN sync_lsn; int64_t release_buffered, release_bytes; + uint64_t fsync_duration_usecs; int yield_count; bool locked; @@ -1408,9 +1431,15 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) "/%" PRIu32, log->log_dir_fh->name, sync_lsn.l.file, sync_lsn.l.offset)); + WT_ERR(__wt_epoch(session, &fsync_start)); WT_ERR(__wt_fsync(session, log->log_dir_fh, true)); + WT_ERR(__wt_epoch(session, &fsync_stop)); + fsync_duration_usecs = + WT_TIMEDIFF_US(fsync_stop, fsync_start); log->sync_dir_lsn = sync_lsn; WT_STAT_FAST_CONN_INCR(session, log_sync_dir); + WT_STAT_FAST_CONN_INCRV(session, + log_sync_dir_duration, fsync_duration_usecs); } /* @@ -1424,7 +1453,13 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) log->log_fh->name, sync_lsn.l.file, sync_lsn.l.offset)); WT_STAT_FAST_CONN_INCR(session, log_sync); + WT_ERR(__wt_epoch(session, &fsync_start)); WT_ERR(__wt_fsync(session, log->log_fh, true)); + WT_ERR(__wt_epoch(session, &fsync_stop)); + fsync_duration_usecs = + WT_TIMEDIFF_US(fsync_stop, fsync_start); + WT_STAT_FAST_CONN_INCRV(session, + log_sync_duration, fsync_duration_usecs); log->sync_lsn = sync_lsn; WT_ERR(__wt_cond_signal(session, log->log_sync_cond)); } @@ -1750,7 +1785,7 @@ err: WT_STAT_FAST_CONN_INCR(session, log_scans); * an error recovery is likely going to fail. Try to provide * a helpful failure message. */ - if (ret != 0 && F_ISSET(log, WT_LOG_NOT_VERIFIED)) { + if (ret != 0 && firstrecord) { __wt_errx(session, "WiredTiger is unable to read the recovery log."); __wt_errx(session, "This may be due to the log" @@ -1760,7 +1795,6 @@ err: WT_STAT_FAST_CONN_INCR(session, log_scans); " opened the database with the correct options including" " all encryption and compression options"); } - F_CLR(log, WT_LOG_NOT_VERIFIED); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); @@ -1832,9 +1866,10 @@ __wt_log_write(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, /* * An error during opening the logging subsystem can result in it * being enabled, but without an open log file. In that case, - * just return. + * just return. We can also have logging opened for reading in a + * read-only database and attempt to write a record on close. */ - if (log->log_fh == NULL) + if (!F_ISSET(log, WT_LOG_OPENED) || F_ISSET(conn, WT_CONN_READONLY)) return (0); ip = record; if ((compressor = conn->log_compressor) != NULL && diff --git a/src/log/log_auto.c b/src/log/log_auto.c index d4dab4e1a33..34bccd0ede4 100644 --- a/src/log/log_auto.c +++ b/src/log/log_auto.c @@ -44,7 +44,7 @@ __wt_logop_read(WT_SESSION_IMPL *session, } static size_t -__logrec_json_unpack_str(char *dest, size_t destlen, const char *src, +__logrec_json_unpack_str(char *dest, size_t destlen, const u_char *src, size_t srclen) { size_t total; diff --git a/src/log/log_slot.c b/src/log/log_slot.c index b7efb1d9018..47071211450 100644 --- a/src/log/log_slot.c +++ b/src/log/log_slot.c @@ -94,6 +94,17 @@ retry: if (WT_LOG_SLOT_DONE(new_state)) *releasep = 1; slot->slot_end_lsn = slot->slot_start_lsn; + /* + * A thread setting the unbuffered flag sets the unbuffered size after + * setting the flag. There could be a delay between a thread setting + * the flag, a thread closing the slot, and the original thread setting + * that value. If the state is unbuffered, wait for the unbuffered + * size to be set. + */ + while (WT_LOG_SLOT_UNBUFFERED_ISSET(old_state) && + slot->slot_unbuffered == 0) + __wt_yield(); + end_offset = WT_LOG_SLOT_JOINED_BUFFERED(old_state) + slot->slot_unbuffered; slot->slot_end_lsn.l.offset += (uint32_t)end_offset; diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 821a996c38b..c19f42327be 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -289,7 +289,8 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, } /* Stop if a running transaction needs the chunk. */ - WT_RET(__wt_txn_update_oldest(session, true)); + WT_RET(__wt_txn_update_oldest( + session, WT_TXN_OLDEST_STRICT | WT_TXN_OLDEST_WAIT)); if (chunk->switch_txn == WT_TXN_NONE || !__wt_txn_visible_all(session, chunk->switch_txn)) { WT_RET(__wt_verbose(session, WT_VERB_LSM, diff --git a/src/meta/meta_track.c b/src/meta/meta_track.c index 4fe628e319b..eb06b2bed66 100644 --- a/src/meta/meta_track.c +++ b/src/meta/meta_track.c @@ -153,7 +153,6 @@ __meta_track_apply(WT_SESSION_IMPL *session, WT_META_TRACK *trk) case WT_ST_REMOVE: case WT_ST_SET: break; - WT_ILLEGAL_VALUE(session); } __meta_track_clear(session, trk); @@ -215,7 +214,6 @@ __meta_track_unroll(WT_SESSION_IMPL *session, WT_META_TRACK *trk) __wt_err(session, ret, "metadata unroll update %s to %s", trk->a, trk->b); break; - WT_ILLEGAL_VALUE(session); } __meta_track_clear(session, trk); diff --git a/src/os_posix/os_errno.c b/src/os_common/os_errno.c index a0f1202c6ef..a8e56b7f1aa 100644 --- a/src/os_posix/os_errno.c +++ b/src/os_common/os_errno.c @@ -23,24 +23,8 @@ __wt_errno(void) } /* - * __wt_map_error_rdonly -- - * Map an error into a WiredTiger error code specific for - * read-only operation which intercepts based on certain types - * of failures. - */ -int -__wt_map_error_rdonly(int error) -{ - if (error == ENOENT) - return (WT_NOTFOUND); - else if (error == EACCES) - return (WT_PERM_DENIED); - return (error); -} - -/* * __wt_strerror -- - * POSIX implementation of WT_SESSION.strerror and wiredtiger_strerror. + * WT_SESSION.strerror and wiredtiger_strerror. */ const char * __wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) @@ -69,3 +53,28 @@ __wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) /* Defeated. */ return ("Unable to return error string"); } + +/* + * __wt_ext_map_windows_error -- + * Extension API call to map a Windows system error to a POSIX/ANSI error. + */ +int +__wt_ext_map_windows_error( + WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, uint32_t windows_error) +{ + WT_UNUSED(wt_api); + WT_UNUSED(wt_session); + + /* + * This extension API only makes sense in Windows builds, but it's hard + * to exclude it otherwise (there's no way to return an error, anyway). + * Call an underlying function on Windows, else panic so callers figure + * out what they're doing wrong. + */ +#ifdef _WIN32 + return (__wt_map_windows_error(windows_error)); +#else + WT_UNUSED(windows_error); + return (WT_PANIC); +#endif +} diff --git a/src/os_common/os_fhandle.c b/src/os_common/os_fhandle.c index 50404c8b354..81e4cc14ccb 100644 --- a/src/os_common/os_fhandle.c +++ b/src/os_common/os_fhandle.c @@ -27,14 +27,15 @@ __fhandle_method_finalize( /* not required: fadvise */ /* not required: fallocate */ /* not required: fallocate_nolock */ - /* not required: lock */ + WT_HANDLE_METHOD_REQ(fh_lock); /* not required: map */ /* not required: map_discard */ /* not required: map_preload */ /* not required: map_unmap */ WT_HANDLE_METHOD_REQ(fh_read); WT_HANDLE_METHOD_REQ(fh_size); - /* not required: sync */ + if (!readonly) + WT_HANDLE_METHOD_REQ(fh_sync); /* not required: sync_nowait */ if (!readonly) { WT_HANDLE_METHOD_REQ(fh_truncate); diff --git a/src/os_common/os_fs_inmemory.c b/src/os_common/os_fs_inmemory.c index 3e4fe628892..09c2e08db83 100644 --- a/src/os_common/os_fs_inmemory.c +++ b/src/os_common/os_fs_inmemory.c @@ -80,22 +80,6 @@ __im_handle_remove(WT_SESSION_IMPL *session, } /* - * __im_handle_size -- - * Return the handle's data size. - */ -static void -__im_handle_size(WT_FILE_HANDLE_INMEM *im_fh, wt_off_t *sizep) -{ - /* - * XXX - * This function exists as a place for this comment. MongoDB assumes - * any file with content will have a non-zero size. In memory tables - * generally are zero-sized, make MongoDB happy. - */ - *sizep = im_fh->buf.size == 0 ? 1024 : (wt_off_t)im_fh->buf.size; -} - -/* * __im_fs_directory_list -- * Return the directory contents. */ @@ -284,7 +268,7 @@ __im_fs_size(WT_FILE_SYSTEM *file_system, if ((im_fh = __im_handle_search(file_system, name)) == NULL) ret = ENOENT; else - __im_handle_size(im_fh, sizep); + *sizep = (wt_off_t)im_fh->buf.size; __wt_spin_unlock(session, &im_fs->lock); @@ -316,6 +300,20 @@ __im_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) } /* + * __im_file_lock -- + * Lock/unlock a file. + */ +static int +__im_file_lock( + WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, bool lock) +{ + WT_UNUSED(file_handle); + WT_UNUSED(wt_session); + WT_UNUSED(lock); + return (0); +} + +/* * __im_file_read -- * POSIX pread. */ @@ -339,7 +337,6 @@ __im_file_read(WT_FILE_HANDLE *file_handle, if (off < im_fh->buf.size) { len = WT_MIN(len, im_fh->buf.size - off); memcpy(buf, (uint8_t *)im_fh->buf.mem + off, len); - im_fh->off = off + len; } else ret = WT_ERROR; @@ -370,7 +367,7 @@ __im_file_size( __wt_spin_lock(session, &im_fs->lock); - __im_handle_size(im_fh, sizep); + *sizep = (wt_off_t)im_fh->buf.size; __wt_spin_unlock(session, &im_fs->lock); @@ -378,6 +375,18 @@ __im_file_size( } /* + * __im_file_sync -- + * In-memory sync. + */ +static int +__im_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) +{ + WT_UNUSED(file_handle); + WT_UNUSED(wt_session); + return (0); +} + +/* * __im_file_truncate -- * POSIX ftruncate. */ @@ -438,7 +447,6 @@ __im_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, memcpy((uint8_t *)im_fh->buf.data + off, buf, len); if (off + len > im_fh->buf.size) im_fh->buf.size = off + len; - im_fh->off = off + len; err: __wt_spin_unlock(session, &im_fs->lock); if (ret == 0) @@ -486,7 +494,6 @@ __im_file_open(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, "%s: file-open: already open", name); im_fh->ref = 1; - im_fh->off = 0; *file_handlep = (WT_FILE_HANDLE *)im_fh; @@ -504,7 +511,6 @@ __im_file_open(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, /* Initialize private information. */ im_fh->ref = 1; - im_fh->off = 0; hash = __wt_hash_city64(name, strlen(name)); bucket = hash % WT_HASH_ARRAY_SIZE; @@ -512,8 +518,10 @@ __im_file_open(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, WT_FILE_HANDLE_INSERT(im_fs, im_fh, bucket); file_handle->close = __im_file_close; + file_handle->fh_lock = __im_file_lock; file_handle->fh_read = __im_file_read; file_handle->fh_size = __im_file_size; + file_handle->fh_sync = __im_file_sync; file_handle->fh_truncate = __im_file_truncate; file_handle->fh_write = __im_file_write; diff --git a/src/os_posix/os_dir.c b/src/os_posix/os_dir.c index ea0ca11fa54..768a1324cd8 100644 --- a/src/os_posix/os_dir.c +++ b/src/os_posix/os_dir.c @@ -25,6 +25,7 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION_IMPL *session; size_t dirallocsz; uint32_t count; + int tret; char **entries; WT_UNUSED(file_system); @@ -64,8 +65,15 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, *dirlistp = entries; *countp = count; -err: if (dirp != NULL) - (void)closedir(dirp); +err: if (dirp != NULL) { + WT_SYSCALL(closedir(dirp), tret); + if (tret != 0) { + __wt_err(session, tret, + "%s: directory-list: closedir", directory); + if (ret == 0) + ret = tret; + } + } if (ret == 0) return (0); diff --git a/src/os_posix/os_fs.c b/src/os_posix/os_fs.c index 1cfa8fd2d2d..86fa2e8f117 100644 --- a/src/os_posix/os_fs.c +++ b/src/os_posix/os_fs.c @@ -98,7 +98,7 @@ __posix_directory_sync( ret = __posix_sync(session, fd, path, "directory-sync"); - WT_SYSCALL_RETRY(close(fd), tret); + WT_SYSCALL(close(fd), tret); if (tret != 0) { __wt_err(session, tret, "%s: directory-sync: close", path); if (ret == 0) @@ -124,7 +124,7 @@ __posix_fs_exist(WT_FILE_SYSTEM *file_system, session = (WT_SESSION_IMPL *)wt_session; - WT_SYSCALL_RETRY(stat(name, &sb), ret); + WT_SYSCALL(stat(name, &sb), ret); if (ret == 0) { *existp = true; return (0); @@ -158,7 +158,7 @@ __posix_fs_remove( * where we're not doing any special checking for standards compliance, * using unlink may be marginally safer. */ - WT_SYSCALL_RETRY(unlink(name), ret); + WT_SYSCALL(unlink(name), ret); if (ret == 0) return (0); WT_RET_MSG(session, ret, "%s: file-remove: unlink", name); @@ -186,7 +186,7 @@ __posix_fs_rename(WT_FILE_SYSTEM *file_system, * with the wrong errno (if errno is garbage), or the generic WT_ERROR * return (if errno is 0), but we've done the best we can. */ - WT_SYSCALL_RETRY(rename(from, to) != 0 ? -1 : 0, ret); + WT_SYSCALL(rename(from, to) != 0 ? -1 : 0, ret); if (ret == 0) return (0); WT_RET_MSG(session, ret, "%s to %s: file-rename: rename", from, to); @@ -208,7 +208,7 @@ __posix_fs_size(WT_FILE_SYSTEM *file_system, session = (WT_SESSION_IMPL *)wt_session; - WT_SYSCALL_RETRY(stat(name, &sb), ret); + WT_SYSCALL(stat(name, &sb), ret); if (ret == 0) { *sizep = sb.st_size; return (0); @@ -232,7 +232,7 @@ __posix_file_advise(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, session = (WT_SESSION_IMPL *)wt_session; pfh = (WT_FILE_HANDLE_POSIX *)file_handle; - WT_SYSCALL_RETRY(posix_fadvise(pfh->fd, offset, len, advice), ret); + WT_SYSCALL(posix_fadvise(pfh->fd, offset, len, advice), ret); if (ret == 0) return (0); @@ -268,7 +268,7 @@ __posix_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) /* Close the file handle. */ if (pfh->fd != -1) { - WT_SYSCALL_RETRY(close(pfh->fd), ret); + WT_SYSCALL(close(pfh->fd), ret); if (ret != 0) __wt_err(session, ret, "%s: handle-close: close", file_handle->name); @@ -309,7 +309,7 @@ __posix_file_lock( fl.l_type = lock ? F_WRLCK : F_UNLCK; fl.l_whence = SEEK_SET; - WT_SYSCALL_RETRY(fcntl(pfh->fd, F_SETLK, &fl) == -1 ? -1 : 0, ret); + WT_SYSCALL(fcntl(pfh->fd, F_SETLK, &fl) == -1 ? -1 : 0, ret); if (ret == 0) return (0); WT_RET_MSG(session, ret, "%s: handle-lock: fcntl", file_handle->name); @@ -369,7 +369,7 @@ __posix_file_size( session = (WT_SESSION_IMPL *)wt_session; pfh = (WT_FILE_HANDLE_POSIX *)file_handle; - WT_SYSCALL_RETRY(fstat(pfh->fd, &sb), ret); + WT_SYSCALL(fstat(pfh->fd, &sb), ret); if (ret == 0) { *sizep = sb.st_size; return (0); @@ -617,7 +617,7 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, * interesting. */ if (!pfh->direct_io && file_type == WT_OPEN_FILE_TYPE_DATA) { - WT_SYSCALL_RETRY( + WT_SYSCALL( posix_fadvise(pfh->fd, 0, 0, POSIX_FADV_RANDOM), ret); if (ret != 0) WT_ERR_MSG(session, ret, diff --git a/src/os_posix/os_map.c b/src/os_posix/os_map.c index d89ba4d7c26..b33f6d82e34 100644 --- a/src/os_posix/os_map.c +++ b/src/os_posix/os_map.c @@ -105,7 +105,7 @@ __wt_posix_map_preload(WT_FILE_HANDLE *fh, if (length <= (size_t)conn->page_size) return (0); - WT_SYSCALL_RETRY(posix_madvise(blk, length, POSIX_MADV_WILLNEED), ret); + WT_SYSCALL(posix_madvise(blk, length, POSIX_MADV_WILLNEED), ret); if (ret == 0) return (0); @@ -138,7 +138,7 @@ __wt_posix_map_discard(WT_FILE_HANDLE *fh, blk = (void *)((uintptr_t)map & ~(uintptr_t)(conn->page_size - 1)); length += WT_PTRDIFF(map, blk); - WT_SYSCALL_RETRY(posix_madvise(blk, length, POSIX_MADV_DONTNEED), ret); + WT_SYSCALL(posix_madvise(blk, length, POSIX_MADV_DONTNEED), ret); if (ret == 0) return (0); diff --git a/src/os_posix/os_thread.c b/src/os_posix/os_thread.c index 35a23622ddc..e57a308c9b0 100644 --- a/src/os_posix/os_thread.c +++ b/src/os_posix/os_thread.c @@ -34,7 +34,7 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) { WT_DECL_RET; - WT_SYSCALL_RETRY(pthread_join(tid, NULL), ret); + WT_SYSCALL(pthread_join(tid, NULL), ret); if (ret == 0) return (0); diff --git a/src/os_posix/os_time.c b/src/os_posix/os_time.c index 0e5a1cdadfb..b1b22a8e684 100644 --- a/src/os_posix/os_time.c +++ b/src/os_posix/os_time.c @@ -18,14 +18,14 @@ __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_DECL_RET; #if defined(HAVE_CLOCK_GETTIME) - WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, tsp), ret); + WT_SYSCALL(clock_gettime(CLOCK_REALTIME, tsp), ret); if (ret == 0) return (0); WT_RET_MSG(session, ret, "clock_gettime"); #elif defined(HAVE_GETTIMEOFDAY) struct timeval v; - WT_SYSCALL_RETRY(gettimeofday(&v, NULL), ret); + WT_SYSCALL(gettimeofday(&v, NULL), ret); if (ret == 0) { tsp->tv_sec = v.tv_sec; tsp->tv_nsec = v.tv_usec * WT_THOUSAND; diff --git a/src/os_win/os_dir.c b/src/os_win/os_dir.c index 6f796f6ef7d..dccacc1e446 100644 --- a/src/os_win/os_dir.c +++ b/src/os_win/os_dir.c @@ -17,6 +17,7 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp) { + DWORD windows_error; HANDLE findhandle; WIN32_FIND_DATA finddata; WT_DECL_ITEM(pathbuf); @@ -45,9 +46,13 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system, WT_ERR(__wt_buf_fmt(session, pathbuf, "%s\\*", dir_copy)); findhandle = FindFirstFileA(pathbuf->data, &finddata); - if (findhandle == INVALID_HANDLE_VALUE) - WT_ERR_MSG(session, __wt_getlasterror(), - "%s: directory-list: FindFirstFile", pathbuf->data); + if (findhandle == INVALID_HANDLE_VALUE) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: directory-list: FindFirstFile: %s", + pathbuf->data, __wt_formatmessage(session, windows_error)); + WT_ERR(__wt_map_windows_error(windows_error)); + } count = 0; do { @@ -74,7 +79,16 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system, *countp = count; err: if (findhandle != INVALID_HANDLE_VALUE) - (void)FindClose(findhandle); + if (FindClose(findhandle) == 0) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: directory-list: FindClose: %s", + pathbuf->data, + __wt_formatmessage(session, windows_error)); + if (ret == 0) + ret = __wt_map_windows_error(windows_error); + } + __wt_free(session, dir_copy); __wt_scr_free(session, &pathbuf); diff --git a/src/os_win/os_dlopen.c b/src/os_win/os_dlopen.c index 9289c8f6488..3da47bf23a3 100644 --- a/src/os_win/os_dlopen.c +++ b/src/os_win/os_dlopen.c @@ -15,6 +15,7 @@ int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) { + DWORD windows_error; WT_DECL_RET; WT_DLH *dlh; @@ -26,9 +27,11 @@ __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) if (path == NULL) { if (GetModuleHandleExA( 0, NULL, (HMODULE *)&dlh->handle) == FALSE) { - ret = __wt_getlasterror(); - WT_ERR_MSG(session, ret, - "GetModuleHandleEx(%s): %s", path, 0); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "GetModuleHandleEx: %s: %s", + path, __wt_formatmessage(session, windows_error)); + WT_ERR(__wt_map_windows_error(windows_error)); } } else { // TODO: load dll here @@ -51,14 +54,20 @@ int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret) { + DWORD windows_error; void *sym; *(void **)sym_ret = NULL; sym = GetProcAddress(dlh->handle, name); - if (sym == NULL && fail) - WT_RET_MSG(session, __wt_getlasterror(), - "GetProcAddress(%s in %s)", name, dlh->name); + if (sym == NULL && fail) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "GetProcAddress: %s in %s: %s", + name, dlh->name, + __wt_formatmessage(session, windows_error)); + WT_RET(__wt_map_windows_error(windows_error)); + } *(void **)sym_ret = sym; return (0); @@ -71,11 +80,14 @@ __wt_dlsym(WT_SESSION_IMPL *session, int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) { + DWORD windows_error; WT_DECL_RET; if (FreeLibrary(dlh->handle) == FALSE) { - ret = __wt_getlasterror(); - __wt_err(session, ret, "FreeLibrary: %s", dlh->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, "FreeLibrary: %s: %s", + dlh->name, __wt_formatmessage(session, windows_error)); + ret = __wt_map_windows_error(windows_error); } __wt_free(session, dlh->name); diff --git a/src/os_win/os_errno.c b/src/os_win/os_errno.c deleted file mode 100644 index f3fffd5ef42..00000000000 --- a/src/os_win/os_errno.c +++ /dev/null @@ -1,151 +0,0 @@ -/*- - * Copyright (c) 2014-2016 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -static const int windows_error_offset = -29000; - -/* - * __wt_map_error_to_windows_error -- - * Return a negative integer, an encoded Windows error - * Standard C errors are positive integers from 0 - ~200 - * Windows errors are from 0 - 15999 according to the documentation - */ -static DWORD -__wt_map_error_to_windows_error(int error) -{ - /* - * Ensure we do not exceed the error range - * Also validate we do not get any COM errors - * (which are negative integers) - */ - WT_ASSERT(NULL, error < 0); - - return (error + -(windows_error_offset)); -} - -/* - * __wt_map_windows_error_to_error -- - * Return a positive integer, a decoded Windows error - */ -static int -__wt_map_windows_error_to_error(DWORD winerr) -{ - return (winerr + windows_error_offset); -} - -/* - * __wt_map_error_rdonly -- - * Map an error into a WiredTiger error code specific for - * read-only operation which intercepts based on certain types - * of failures. - */ -int -__wt_map_error_rdonly(int error) -{ - if (error == ERROR_FILE_NOT_FOUND) - return (WT_NOTFOUND); - else if (error == ERROR_ACCESS_DENIED) - return (WT_PERM_DENIED); - return (error); -} - -/* - * __wt_errno -- - * Return errno, or WT_ERROR if errno not set. - */ -int -__wt_errno(void) -{ - /* - * Check for 0: - * It's easy to introduce a problem by calling the wrong error function, - * for example, this function when the MSVC function set the C runtime - * error value. Handle gracefully and always return an error. - */ - return (errno == 0 ? WT_ERROR : errno); -} - -/* - * __wt_getlasterror -- - * Return GetLastError, or WT_ERROR if error not set. - */ -int -__wt_getlasterror(void) -{ - /* - * Called when we know an error occurred, and we want the system - * error code. - */ - DWORD err = GetLastError(); - - /* - * Check for ERROR_SUCCESS: - * It's easy to introduce a problem by calling the wrong error function, - * for example, this function when the MSVC function set the C runtime - * error value. Handle gracefully and always return an error. - */ - return (err == ERROR_SUCCESS ? - WT_ERROR : __wt_map_windows_error_to_error(err)); -} - -/* - * __wt_strerror -- - * Windows implementation of WT_SESSION.strerror and wiredtiger_strerror. - */ -const char * -__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) -{ - DWORD lasterror; - const char *p; - char buf[512]; - - /* - * Check for a WiredTiger or POSIX constant string, no buffer needed. - */ - if ((p = __wt_wiredtiger_error(error)) != NULL) - return (p); - - /* - * When called from wiredtiger_strerror, write a passed-in buffer. - * When called from WT_SESSION.strerror, write the session's buffer. - * - * Check for Windows errors. - */ - if (error < 0) { - error = __wt_map_error_to_windows_error(error); - - lasterror = FormatMessageA( - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, /* let system choose the correct LANGID */ - buf, - sizeof(buf), - NULL); - - if (lasterror != 0 && session == NULL && - snprintf(errbuf, errlen, "%s", buf) > 0) - return (errbuf); - if (lasterror != 0 && session != NULL && - __wt_buf_fmt(session, &session->err, "%s", buf) == 0) - return (session->err.data); - } - - /* Fallback to a generic message. */ - if (session == NULL && - snprintf(errbuf, errlen, "error return: %d", error) > 0) - return (errbuf); - if (session != NULL && __wt_buf_fmt( - session, &session->err, "error return: %d", error) == 0) - return (session->err.data); - - /* Defeated. */ - return ("Unable to return error string"); -} diff --git a/src/os_win/os_fs.c b/src/os_win/os_fs.c index 4da60d5ffb0..5daba124e90 100644 --- a/src/os_win/os_fs.c +++ b/src/os_win/os_fs.c @@ -39,17 +39,20 @@ static int __win_fs_remove( WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name) { - WT_DECL_RET; + DWORD windows_error; WT_SESSION_IMPL *session; WT_UNUSED(file_system); session = (WT_SESSION_IMPL *)wt_session; - if (DeleteFileA(name) == FALSE) - WT_RET_MSG(session, __wt_getlasterror(), - "%s: file-remove: DeleteFileA", name); - + if (DeleteFileA(name) == FALSE) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: file-remove: DeleteFileA: %s", + name, __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); + } return (0); } @@ -61,7 +64,7 @@ static int __win_fs_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *from, const char *to) { - WT_DECL_RET; + DWORD windows_error; WT_SESSION_IMPL *session; WT_UNUSED(file_system); @@ -73,13 +76,21 @@ __win_fs_rename(WT_FILE_SYSTEM *file_system, * it exists. */ if (GetFileAttributesA(to) != INVALID_FILE_ATTRIBUTES) - if (DeleteFileA(to) == FALSE) - WT_RET_MSG(session, __wt_getlasterror(), - "%s to %s: file-rename: rename", from, to); + if (DeleteFileA(to) == FALSE) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: file-rename: DeleteFileA: %s", + to, __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); + } - if (MoveFileA(from, to) == FALSE) - WT_RET_MSG(session, __wt_getlasterror(), - "%s to %s: file-rename: rename", from, to); + if (MoveFileA(from, to) == FALSE) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s to %s: file-rename: MoveFileA: %s", + from, to, __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); + } return (0); } @@ -92,6 +103,7 @@ int __wt_win_fs_size(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, wt_off_t *sizep) { + DWORD windows_error; WIN32_FILE_ATTRIBUTE_DATA data; WT_SESSION_IMPL *session; @@ -105,8 +117,11 @@ __wt_win_fs_size(WT_FILE_SYSTEM *file_system, return (0); } - WT_RET_MSG(session, __wt_getlasterror(), - "%s: file-size: GetFileAttributesEx", name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: file-size: GetFileAttributesEx: %s", + name, __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } /* @@ -116,6 +131,7 @@ __wt_win_fs_size(WT_FILE_SYSTEM *file_system, static int __win_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) { + DWORD windows_error; WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; @@ -132,17 +148,22 @@ __win_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) */ if (win_fh->filehandle != INVALID_HANDLE_VALUE && CloseHandle(win_fh->filehandle) == 0) { - ret = __wt_getlasterror(); - __wt_err(session, ret, - "%s: handle-close: CloseHandle", file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: handle-close: CloseHandle: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + ret = __wt_map_windows_error(windows_error); } if (win_fh->filehandle_secondary != INVALID_HANDLE_VALUE && CloseHandle(win_fh->filehandle_secondary) == 0) { - ret = __wt_getlasterror(); - __wt_err(session, ret, - "%s: handle-close: secondary: CloseHandle", - file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: handle-close: secondary: CloseHandle: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + ret = __wt_map_windows_error(windows_error); } __wt_free(session, file_handle->name); @@ -158,7 +179,7 @@ static int __win_file_lock( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, bool lock) { - WT_DECL_RET; + DWORD windows_error; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; @@ -177,17 +198,23 @@ __win_file_lock( */ if (lock) { if (LockFile(win_fh->filehandle, 0, 0, 1, 0) == FALSE) { - ret = __wt_getlasterror(); - __wt_err(session, ret, - "%s: handle-lock: LockFile", file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: handle-lock: LockFile: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } } else if (UnlockFile(win_fh->filehandle, 0, 0, 1, 0) == FALSE) { - ret = __wt_getlasterror(); - __wt_err(session, ret, - "%s: handle-lock: UnlockFile", file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: handle-lock: UnlockFile: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } - return (ret); + return (0); } /* @@ -198,7 +225,7 @@ static int __win_file_read(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset, size_t len, void *buf) { - DWORD chunk, nr; + DWORD chunk, nr, windows_error; uint8_t *addr; OVERLAPPED overlapped = { 0 }; WT_FILE_HANDLE_WIN *win_fh; @@ -225,12 +252,15 @@ __win_file_read(WT_FILE_HANDLE *file_handle, overlapped.OffsetHigh = UINT32_MAX & (offset >> 32); if (!ReadFile( - win_fh->filehandle, addr, chunk, &nr, &overlapped)) - WT_RET_MSG(session, - __wt_getlasterror(), + win_fh->filehandle, addr, chunk, &nr, &overlapped)) { + windows_error = __wt_getlasterror(); + __wt_errx(session, "%s: handle-read: ReadFile: failed to read %lu " - "bytes at offset %" PRIuMAX, - file_handle->name, chunk, (uintmax_t)offset); + "bytes at offset %" PRIuMAX ": %s", + file_handle->name, chunk, (uintmax_t)offset, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); + } } return (0); } @@ -243,6 +273,7 @@ static int __win_file_size( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t *sizep) { + DWORD windows_error; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; LARGE_INTEGER size; @@ -255,8 +286,11 @@ __win_file_size( return (0); } - WT_RET_MSG(session, __wt_getlasterror(), - "%s: handle-size: GetFileSizeEx", file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: handle-size: GetFileSizeEx: %s", + file_handle->name, __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } /* @@ -266,7 +300,7 @@ __win_file_size( static int __win_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) { - WT_DECL_RET; + DWORD windows_error; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; @@ -283,10 +317,12 @@ __win_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) return (0); if (FlushFileBuffers(win_fh->filehandle) == FALSE) { - ret = __wt_getlasterror(); - WT_RET_MSG(session, ret, - "%s handle-sync: FlushFileBuffers error", - file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s handle-sync: FlushFileBuffers: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } return (0); } @@ -299,7 +335,7 @@ static int __win_file_truncate( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t len) { - WT_DECL_RET; + DWORD windows_error; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; LARGE_INTEGER largeint; @@ -313,18 +349,25 @@ __win_file_truncate( WT_RET_MSG(session, EINVAL, "%s: handle-truncate: read-only", file_handle->name); - if (SetFilePointerEx( - win_fh->filehandle_secondary, largeint, NULL, FILE_BEGIN) == FALSE) - WT_RET_MSG(session, __wt_getlasterror(), - "%s: handle-truncate: SetFilePointerEx", - file_handle->name); + if (SetFilePointerEx(win_fh->filehandle_secondary, + largeint, NULL, FILE_BEGIN) == FALSE) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: handle-truncate: SetFilePointerEx: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); + } if (SetEndOfFile(win_fh->filehandle_secondary) == FALSE) { if (GetLastError() == ERROR_USER_MAPPED_FILE) return (EBUSY); - WT_RET_MSG(session, __wt_getlasterror(), - "%s: handle-truncate: SetEndOfFile error", - file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: handle-truncate: SetEndOfFile: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } return (0); } @@ -337,8 +380,7 @@ static int __win_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset, size_t len, const void *buf) { - DWORD chunk; - DWORD nw; + DWORD chunk, nw, windows_error; const uint8_t *addr; OVERLAPPED overlapped = { 0 }; WT_FILE_HANDLE_WIN *win_fh; @@ -365,11 +407,15 @@ __win_file_write(WT_FILE_HANDLE *file_handle, overlapped.OffsetHigh = UINT32_MAX & (offset >> 32); if (!WriteFile( - win_fh->filehandle, addr, chunk, &nw, &overlapped)) - WT_RET_MSG(session, __wt_getlasterror(), + win_fh->filehandle, addr, chunk, &nw, &overlapped)) { + windows_error = __wt_getlasterror(); + __wt_errx(session, "%s: handle-write: WriteFile: failed to write %lu " - "bytes at offset %" PRIuMAX, - file_handle->name, chunk, (uintmax_t)offset); + "bytes at offset %" PRIuMAX ": %s", + file_handle->name, chunk, (uintmax_t)offset, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); + } } return (0); } @@ -383,7 +429,7 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, WT_OPEN_FILE_TYPE file_type, uint32_t flags, WT_FILE_HANDLE **file_handlep) { - DWORD dwCreationDisposition; + DWORD dwCreationDisposition, windows_error; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_FILE_HANDLE *file_handle; @@ -464,13 +510,17 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, win_fh->filehandle = CreateFileA(name, desired_access, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, f, NULL); - if (win_fh->filehandle == INVALID_HANDLE_VALUE) - WT_ERR_MSG(session, __wt_getlasterror(), + if (win_fh->filehandle == INVALID_HANDLE_VALUE) { + windows_error = __wt_getlasterror(); + __wt_errx(session, win_fh->direct_io ? "%s: handle-open: CreateFileA: failed with direct " "I/O configured, some filesystem types do not " - "support direct I/O" : - "%s: handle-open: CreateFileA", name); + "support direct I/O: %s" : + "%s: handle-open: CreateFileA: %s", + name, __wt_formatmessage(session, windows_error)); + WT_ERR(__wt_map_windows_error(windows_error)); + } } /* @@ -482,9 +532,13 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, win_fh->filehandle_secondary = CreateFileA(name, desired_access, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, f, NULL); - if (win_fh->filehandle_secondary == INVALID_HANDLE_VALUE) - WT_ERR_MSG(session, __wt_getlasterror(), - "%s: handle-open: CreateFileA: secondary", name); + if (win_fh->filehandle_secondary == INVALID_HANDLE_VALUE) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: handle-open: CreateFileA: secondary: %s", + name, __wt_formatmessage(session, windows_error)); + WT_ERR(__wt_map_windows_error(windows_error)); + } } directory_open: diff --git a/src/os_win/os_getenv.c b/src/os_win/os_getenv.c index 9b297ac3a74..fe228328ee6 100644 --- a/src/os_win/os_getenv.c +++ b/src/os_win/os_getenv.c @@ -15,22 +15,22 @@ int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) { - WT_DECL_RET; - DWORD size; + DWORD size, windows_error; *envp = NULL; - size = GetEnvironmentVariableA(variable, NULL, 0); - if (size <= 1) + if ((size = GetEnvironmentVariableA(variable, NULL, 0)) <= 1) return (WT_NOTFOUND); - WT_RET(__wt_calloc(session, 1, size, envp)); + WT_RET(__wt_malloc(session, (size_t)size, envp)); - ret = GetEnvironmentVariableA(variable, *envp, size); /* We expect the number of bytes not including nul terminator. */ - if ((ret + 1) != size) - WT_RET_MSG(session, __wt_getlasterror(), - "GetEnvironmentVariableA failed: %s", variable); + if (GetEnvironmentVariableA(variable, *envp, size) == size - 1) + return (0); - return (0); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "GetEnvironmentVariableA: %s: %s", + variable, __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } diff --git a/src/os_win/os_map.c b/src/os_win/os_map.c index 488cbfb2ceb..8f5b289062d 100644 --- a/src/os_win/os_map.c +++ b/src/os_win/os_map.c @@ -16,7 +16,7 @@ int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) { - WT_DECL_RET; + DWORD windows_error; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; size_t len; @@ -40,18 +40,27 @@ __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, mapped_cookie = CreateFileMappingA( win_fh->filehandle, NULL, PAGE_READONLY, 0, 0, NULL); - if (mapped_cookie == NULL) - WT_RET_MSG(session, __wt_getlasterror(), - "%s: memory-map: CreateFileMappingA", file_handle->name); + if (mapped_cookie == NULL) { + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: memory-map: CreateFileMappingA: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); + } if ((map = MapViewOfFile(mapped_cookie, FILE_MAP_READ, 0, 0, len)) == NULL) { /* Retrieve the error before cleaning up. */ - ret = __wt_getlasterror(); - CloseHandle(mapped_cookie); + windows_error = __wt_getlasterror(); + + (void)CloseHandle(mapped_cookie); - WT_RET_MSG(session, ret, - "%s: memory-map: MapViewOfFile", file_handle->name); + __wt_errx(session, + "%s: memory-map: MapViewOfFile: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } *(void **)mapped_cookiep = mapped_cookie; @@ -68,6 +77,7 @@ int __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_region, size_t length, void *mapped_cookie) { + DWORD windows_error; WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; @@ -80,15 +90,21 @@ __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, file_handle->name, length); if (UnmapViewOfFile(mapped_region) == 0) { - ret = __wt_getlasterror(); - __wt_err(session, ret, - "%s: memory-unmap: UnmapViewOfFile", file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: memory-unmap: UnmapViewOfFile: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + ret = __wt_map_windows_error(windows_error); } if (CloseHandle(*(void **)mapped_cookie) == 0) { - ret = __wt_getlasterror(); - __wt_err(session, ret, - "%s: memory-unmap: CloseHandle", file_handle->name); + windows_error = __wt_getlasterror(); + __wt_errx(session, + "%s: memory-unmap: CloseHandle: %s", + file_handle->name, + __wt_formatmessage(session, windows_error)); + ret = __wt_map_windows_error(windows_error); } return (ret); diff --git a/src/os_win/os_mtx_cond.c b/src/os_win/os_mtx_cond.c index af4a5035076..8645fdaccb3 100644 --- a/src/os_win/os_mtx_cond.c +++ b/src/os_win/os_mtx_cond.c @@ -45,10 +45,10 @@ int __wt_cond_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled) { - DWORD err, milliseconds; - WT_DECL_RET; - uint64_t milliseconds64; + BOOL sleepret; + DWORD milliseconds, windows_error; bool locked; + uint64_t milliseconds64; locked = false; @@ -88,33 +88,35 @@ __wt_cond_wait_signal( if (milliseconds == 0) milliseconds = 1; - ret = SleepConditionVariableCS( + sleepret = SleepConditionVariableCS( &cond->cond, &cond->mtx, milliseconds); } else - ret = SleepConditionVariableCS( + sleepret = SleepConditionVariableCS( &cond->cond, &cond->mtx, INFINITE); /* * SleepConditionVariableCS returns non-zero on success, 0 on timeout - * or failure. Check for timeout, else convert to a WiredTiger error - * value and fail. + * or failure. */ - if (ret == 0) { - if ((err = GetLastError()) == ERROR_TIMEOUT) + if (sleepret == 0) { + windows_error = __wt_getlasterror(); + if (windows_error == ERROR_TIMEOUT) { *signalled = false; - else - ret = __wt_getlasterror(); - } else - ret = 0; + sleepret = 1; + } + } (void)__wt_atomic_subi32(&cond->waiters, 1); if (locked) LeaveCriticalSection(&cond->mtx); - if (ret == 0) + if (sleepret != 0) return (0); - WT_RET_MSG(session, ret, "SleepConditionVariableCS"); + + __wt_errx(session, "SleepConditionVariableCS: %s", + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } /* diff --git a/src/os_win/os_path.c b/src/os_win/os_path.c index e9532de2b38..220752ce7a1 100644 --- a/src/os_win/os_path.c +++ b/src/os_win/os_path.c @@ -19,7 +19,7 @@ __wt_absolute_path(const char *path) * Check for a drive name (for example, "D:"), allow both forward and * backward slashes. */ - if (strlen(path) >= 3 && isalpha(path[0]) && path[1] == ':') + if (strlen(path) >= 3 && __wt_isalpha(path[0]) && path[1] == ':') path += 2; return (path[0] == '/' || path[0] == '\\'); } diff --git a/src/os_win/os_thread.c b/src/os_win/os_thread.c index 2b846216442..a34dff776b6 100644 --- a/src/os_win/os_thread.c +++ b/src/os_win/os_thread.c @@ -31,19 +31,24 @@ __wt_thread_create(WT_SESSION_IMPL *session, int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) { - WT_DECL_RET; + DWORD windows_error; - if ((ret = WaitForSingleObject(tid, INFINITE)) != WAIT_OBJECT_0) - /* - * If we fail to wait, we will leak handles so do not continue - */ - WT_PANIC_RET(session, - ret == WAIT_FAILED ? __wt_getlasterror() : ret, - "thread join: WaitForSingleObject"); + if ((windows_error = + WaitForSingleObject(tid, INFINITE)) != WAIT_OBJECT_0) { + if (windows_error == WAIT_FAILED) + windows_error = __wt_getlasterror(); + __wt_errx(session, "thread join: WaitForSingleObject: %s", + __wt_formatmessage(session, windows_error)); + + /* If we fail to wait, we will leak handles, do not continue. */ + return (WT_PANIC); + } if (CloseHandle(tid) == 0) { - WT_RET_MSG(session, - __wt_getlasterror(), "thread join: CloseHandle"); + windows_error = __wt_getlasterror(); + __wt_errx(session, "thread join: CloseHandle: %s", + __wt_formatmessage(session, windows_error)); + return (__wt_map_windows_error(windows_error)); } return (0); diff --git a/src/os_win/os_winerr.c b/src/os_win/os_winerr.c new file mode 100644 index 00000000000..70499580c48 --- /dev/null +++ b/src/os_win/os_winerr.c @@ -0,0 +1,130 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_getlasterror -- + * Return GetLastError, or a relatively generic Windows error if the system + * error code isn't set. + */ +DWORD +__wt_getlasterror(void) +{ + DWORD windows_error; + + /* + * Check for ERROR_SUCCESS: + * It's easy to introduce a problem by calling the wrong error function, + * for example, this function when the MSVC function set the C runtime + * error value. Handle gracefully and always return an error. + */ + windows_error = GetLastError(); + return (windows_error == ERROR_SUCCESS ? + ERROR_INVALID_PARAMETER : windows_error); +} + +/* + * __wt_map_windows_error -- + * Map Windows errors to POSIX/ANSI errors. + */ +int +__wt_map_windows_error(DWORD windows_error) +{ + static const struct { + int windows_error; + int posix_error; + } list[] = { + { ERROR_ACCESS_DENIED, EACCES }, + { ERROR_ALREADY_EXISTS, EEXIST }, + { ERROR_ARENA_TRASHED, EFAULT }, + { ERROR_BAD_COMMAND, EFAULT }, + { ERROR_BAD_ENVIRONMENT, EFAULT }, + { ERROR_BAD_FORMAT, EFAULT }, + { ERROR_BAD_NETPATH, ENOENT }, + { ERROR_BAD_NET_NAME, ENOENT }, + { ERROR_BAD_PATHNAME, ENOENT }, + { ERROR_BROKEN_PIPE, EPIPE }, + { ERROR_CANNOT_MAKE, EACCES }, + { ERROR_CHILD_NOT_COMPLETE, ECHILD }, + { ERROR_CURRENT_DIRECTORY, EACCES }, + { ERROR_DIRECT_ACCESS_HANDLE, EBADF }, + { ERROR_DIR_NOT_EMPTY, ENOTEMPTY }, + { ERROR_DISK_FULL, ENOSPC }, + { ERROR_DRIVE_LOCKED, EACCES }, + { ERROR_FAIL_I24, EACCES }, + { ERROR_FILENAME_EXCED_RANGE, ENOENT }, + { ERROR_FILE_EXISTS, EEXIST }, + { ERROR_FILE_NOT_FOUND, ENOENT }, + { ERROR_GEN_FAILURE, EFAULT }, + { ERROR_INVALID_ACCESS, EACCES }, + { ERROR_INVALID_BLOCK, EFAULT }, + { ERROR_INVALID_DATA, EFAULT }, + { ERROR_INVALID_DRIVE, ENOENT }, + { ERROR_INVALID_FUNCTION, EINVAL }, + { ERROR_INVALID_HANDLE, EBADF }, + { ERROR_INVALID_PARAMETER, EINVAL }, + { ERROR_INVALID_TARGET_HANDLE, EBADF }, + { ERROR_LOCK_FAILED, EBUSY }, + { ERROR_LOCK_VIOLATION, EBUSY }, + { ERROR_MAX_THRDS_REACHED, EAGAIN }, + { ERROR_NEGATIVE_SEEK, EINVAL }, + { ERROR_NESTING_NOT_ALLOWED, EAGAIN }, + { ERROR_NETWORK_ACCESS_DENIED, EACCES }, + { ERROR_NOT_ENOUGH_MEMORY, ENOMEM }, + { ERROR_NOT_ENOUGH_QUOTA, ENOMEM }, + { ERROR_NOT_LOCKED, EACCES }, + { ERROR_NOT_READY, EBUSY }, + { ERROR_NOT_SAME_DEVICE, EXDEV }, + { ERROR_NO_DATA, EPIPE }, + { ERROR_NO_MORE_FILES, EMFILE }, + { ERROR_NO_PROC_SLOTS, EAGAIN }, + { ERROR_PATH_NOT_FOUND, ENOENT }, + { ERROR_READ_FAULT, EFAULT }, + { ERROR_RETRY, EINTR }, + { ERROR_SEEK_ON_DEVICE, EACCES }, + { ERROR_SHARING_VIOLATION, EBUSY }, + { ERROR_TOO_MANY_OPEN_FILES, EMFILE }, + { ERROR_WAIT_NO_CHILDREN, ECHILD }, + { ERROR_WRITE_FAULT, EFAULT }, + { ERROR_WRITE_PROTECT, EACCES }, + }; + int i; + + for (i = 0; i < WT_ELEMENTS(list); ++i) + if (windows_error == list[i].windows_error) + return (list[i].posix_error); + + /* Untranslatable error, go generic. */ + return (WT_ERROR); +} + +/* + * __wt_formatmessage -- + * Windows error formatting. + */ +const char * +__wt_formatmessage(WT_SESSION_IMPL *session, DWORD windows_error) +{ + /* + * !!! + * This function MUST handle a NULL session handle. + * + * Grow the session error buffer as necessary. + */ + if (session != NULL && + __wt_buf_initsize(session, &session->err, 512) == 0 && + FormatMessageA( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, windows_error, + 0, /* Let system choose the correct LANGID. */ + session->err.mem, (DWORD)512, NULL) != 0) + return (session->err.data); + + return ("Unable to format Windows error string"); +} diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index 6e406fc7180..b49946bb10e 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -1041,6 +1041,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, bool append_origv, skipped; *updp = NULL; + append = NULL; /* -Wconditional-uninitialized */ btree = S2BT(session); page = r->page; @@ -2428,7 +2429,7 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); break; case SPLIT_TRACKING_RAW: - WT_ILLEGAL_VALUE(session); + return (__wt_illegal_value(session, NULL)); } /* @@ -2959,7 +2960,6 @@ __rec_split_finish_std(WT_SESSION_IMPL *session, WT_RECONCILE *r) * wrote anything, or there's a remaindered block of data. */ break; - WT_ILLEGAL_VALUE(session); } /* @@ -3330,6 +3330,8 @@ supd_check_complete: } bnd->entries = r->entries; + +#ifdef HAVE_VERBOSE /* Output a verbose message if we create a page without many entries */ if (WT_VERBOSE_ISSET(session, WT_VERB_SPLIT) && r->entries < 6) WT_ERR(__wt_verbose(session, WT_VERB_SPLIT, @@ -3339,6 +3341,7 @@ supd_check_complete: r->entries, r->page->memory_footprint, r->bnd_next, F_ISSET(r, WT_EVICTING) ? "evict" : "checkpoint", r->bnd_state)); +#endif WT_ERR(__wt_bt_write(session, buf, addr, &addr_size, false, bnd->already_compressed)); @@ -3527,6 +3530,7 @@ __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) r = cbulk->reconcile; r->is_bulk_load = true; + recno = WT_RECNO_OOB; /* -Werror=maybe-uninitialized */ switch (btree->type) { case BTREE_COL_FIX: case BTREE_COL_VAR: @@ -3535,7 +3539,6 @@ __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) case BTREE_ROW: recno = WT_RECNO_OOB; break; - WT_ILLEGAL_VALUE(session); } return (__rec_split_init( @@ -3569,7 +3572,6 @@ __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) break; case BTREE_ROW: break; - WT_ILLEGAL_VALUE(session); } WT_RET(__rec_split_finish(session, r)); diff --git a/src/schema/schema_create.c b/src/schema/schema_create.c index 67d64cf1c75..f250612d0ae 100644 --- a/src/schema/schema_create.c +++ b/src/schema/schema_create.c @@ -578,7 +578,7 @@ __create_table(WT_SESSION_IMPL *session, WT_ERR(EEXIST); exists = true; } - WT_RET_NOTFOUND_OK(ret); + WT_ERR_NOTFOUND_OK(ret); WT_ERR(__wt_config_gets(session, cfg, "colgroups", &cval)); WT_ERR(__wt_config_subinit(session, &conf, &cval)); diff --git a/src/schema/schema_list.c b/src/schema/schema_list.c index 5e9caf94b7a..79e3ef1da7c 100644 --- a/src/schema/schema_list.c +++ b/src/schema/schema_list.c @@ -20,6 +20,8 @@ __schema_add_table(WT_SESSION_IMPL *session, WT_TABLE *table; uint64_t bucket; + table = NULL; /* -Wconditional-uninitialized */ + /* Make sure the metadata is open before getting other locks. */ WT_RET(__wt_metadata_cursor(session, NULL)); diff --git a/src/schema/schema_project.c b/src/schema/schema_project.c index 4d29b2baa13..fd59539ae89 100644 --- a/src/schema/schema_project.c +++ b/src/schema/schema_project.c @@ -353,7 +353,8 @@ __wt_schema_project_slice(WT_SESSION_IMPL *session, WT_CURSOR **cp, /* Make sure the types are compatible. */ WT_ASSERT(session, - tolower(pv.type) == tolower(vpv.type)); + __wt_tolower((u_char)pv.type) == + __wt_tolower((u_char)vpv.type)); pv.u = vpv.u; len = __pack_size(session, &pv); @@ -459,7 +460,8 @@ __wt_schema_project_merge(WT_SESSION_IMPL *session, WT_RET(__pack_next(&vpack, &vpv)); /* Make sure the types are compatible. */ WT_ASSERT(session, - tolower(pv.type) == tolower(vpv.type)); + __wt_tolower((u_char)pv.type) == + __wt_tolower((u_char)vpv.type)); vpv.u = pv.u; len = __pack_size(session, &vpv); WT_RET(__wt_buf_grow(session, diff --git a/src/support/err.c b/src/support/err.c index 815b79c16db..93c0af37328 100644 --- a/src/support/err.c +++ b/src/support/err.c @@ -469,6 +469,9 @@ void __wt_assert(WT_SESSION_IMPL *session, int error, const char *file_name, int line_number, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6))) +#ifdef HAVE_DIAGNOSTIC + WT_GCC_FUNC_ATTRIBUTE((noreturn)) +#endif { va_list ap; @@ -493,7 +496,10 @@ __wt_panic(WT_SESSION_IMPL *session) F_SET(S2C(session), WT_CONN_PANIC); __wt_err(session, WT_PANIC, "the process must exit and restart"); -#if !defined(HAVE_DIAGNOSTIC) +#if defined(HAVE_DIAGNOSTIC) + __wt_abort(session); /* Drop core if testing. */ + /* NOTREACHED */ +#else /* * Chaos reigns within. * Reflect, repent, and reboot. @@ -501,9 +507,6 @@ __wt_panic(WT_SESSION_IMPL *session) */ return (WT_PANIC); #endif - - __wt_abort(session); /* Drop core if testing. */ - /* NOTREACHED */ } /* @@ -517,12 +520,12 @@ __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) name == NULL ? "" : name, name == NULL ? "" : ": ", "encountered an illegal file format or internal value"); -#if !defined(HAVE_DIAGNOSTIC) - return (__wt_panic(session)); -#endif - +#if defined(HAVE_DIAGNOSTIC) __wt_abort(session); /* Drop core if testing. */ /* NOTREACHED */ +#else + return (__wt_panic(session)); +#endif } /* diff --git a/src/support/global.c b/src/support/global.c index e0d5bafeaa8..eba88bf2b20 100644 --- a/src/support/global.c +++ b/src/support/global.c @@ -111,11 +111,13 @@ void __wt_attach(WT_SESSION_IMPL *session) { #ifdef HAVE_ATTACH + u_int i; + __wt_errx(session, "process ID %" PRIdMAX ": waiting for debugger...", (intmax_t)getpid()); /* Sleep forever, the debugger will interrupt us when it attaches. */ - for (;;) + for (i = 0; i < WT_MILLION; ++i) __wt_sleep(10, 0); #else WT_UNUSED(session); diff --git a/src/support/hash_city.c b/src/support/hash_city.c index 7a700aa809c..8354532e820 100644 --- a/src/support/hash_city.c +++ b/src/support/hash_city.c @@ -85,6 +85,7 @@ static uint32_t UNALIGNED_LOAD32(const char *p) { return (result); } +#ifdef WORDS_BIGENDIAN #ifdef _MSC_VER #include <stdlib.h> @@ -132,7 +133,6 @@ static uint32_t UNALIGNED_LOAD32(const char *p) { #endif -#ifdef WORDS_BIGENDIAN #define uint32_in_expected_order(x) (bswap_32(x)) #define uint64_in_expected_order(x) (bswap_64(x)) #else diff --git a/src/support/hazard.c b/src/support/hazard.c index 13e0eb3b9ac..dee85586a4d 100644 --- a/src/support/hazard.c +++ b/src/support/hazard.c @@ -121,7 +121,8 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp return (0); } - __wt_errx(session, "session %p: hazard pointer table full", session); + __wt_errx(session, + "session %p: hazard pointer table full", (void *)session); #ifdef HAVE_DIAGNOSTIC __hazard_dump(session); #endif @@ -176,7 +177,8 @@ __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) * because using a page we didn't have pinned down implies corruption. */ WT_PANIC_RET(session, EINVAL, - "session %p: clear hazard pointer: %p: not found", session, page); + "session %p: clear hazard pointer: %p: not found", + (void *)session, (void *)page); } /* @@ -204,7 +206,8 @@ __wt_hazard_close(WT_SESSION_IMPL *session) return; __wt_errx(session, - "session %p: close hazard pointer table: table not empty", session); + "session %p: close hazard pointer table: table not empty", + (void *)session); #ifdef HAVE_DIAGNOSTIC __hazard_dump(session); @@ -232,7 +235,7 @@ __wt_hazard_close(WT_SESSION_IMPL *session) __wt_errx(session, "session %p: close hazard pointer table: count didn't " "match entries", - session); + (void *)session); } #ifdef HAVE_DIAGNOSTIC @@ -250,6 +253,7 @@ __hazard_dump(WT_SESSION_IMPL *session) if (hp->page != NULL) __wt_errx(session, "session %p: hazard pointer %p: %s, line %d", - session, hp->page, hp->file, hp->line); + (void *)session, + (void *)hp->page, hp->file, hp->line); } #endif diff --git a/src/support/hex.c b/src/support/hex.c index d42a84154ca..5c48ce8b74a 100644 --- a/src/support/hex.c +++ b/src/support/hex.c @@ -84,7 +84,7 @@ __wt_raw_to_esc_hex( WT_RET(__wt_buf_init(session, to, size * 3 + 1)); for (p = from, t = to->mem, i = size; i > 0; --i, ++p) - if (isprint((int)*p)) { + if (__wt_isprint((u_char)*p)) { if (*p == '\\') *t++ = '\\'; *t++ = *p; diff --git a/src/support/huffman.c b/src/support/huffman.c index 1e1aaeab5b5..05612cdbe80 100644 --- a/src/support/huffman.c +++ b/src/support/huffman.c @@ -230,19 +230,19 @@ set_codes(WT_FREQTREE_NODE *node, * lower-order bits for consecutive numbering. */ if (len < MAX_CODE_LENGTH && - ((half = 1 << (remaining - 1)) < node->left->weight || - half < node->right->weight)) { - pattern = pattern << remaining; + ((half = (uint16_t)(1 << (remaining - 1))) < + node->left->weight || half < node->right->weight)) { + pattern = (uint16_t)(pattern << remaining); len = MAX_CODE_LENGTH; } if (len < MAX_CODE_LENGTH) { - patternleft = (pattern << 1) | 0; - patternright = (pattern << 1) | 1; + patternleft = (uint16_t)((pattern << 1) | 0); + patternright = (uint16_t)((pattern << 1) | 1); len++; } else { /* "low bit mode" */ patternleft = pattern; - patternright = pattern + node->left->weight; + patternright = (uint16_t)(pattern + node->left->weight); /* len unchanged */ } @@ -284,12 +284,12 @@ make_table(WT_SESSION_IMPL *session, uint8_t *code2symbol, * than necessary, we allocate (2 ^ max-code-length) of them. */ c = codes[i].pattern; - shift = max_depth - len; + shift = (uint8_t)(max_depth - len); c1 = (uint32_t)c << shift; c2 = (uint32_t)(c + 1) << shift; for (j = c1; j < c2; j++) { WT_ASSERT(session, code2symbol[j] == 0); - code2symbol[j] = i; + code2symbol[j] = (uint8_t)i; } } } @@ -694,7 +694,7 @@ __wt_huffman_encode(WT_SESSION_IMPL *session, void *huffman_arg, * used in the last byte, unless they're 0, in which case there are 8 * bits used in the last byte. */ - padding_info = (bitpos % 8) << (8 - WT_HUFFMAN_HEADER); + padding_info = (uint8_t)((bitpos % 8) << (8 - WT_HUFFMAN_HEADER)); ((uint8_t *)tmp->mem)[0] |= padding_info; /* Copy result of exact known size into caller's buffer. */ @@ -808,11 +808,12 @@ __wt_huffman_decode(WT_SESSION_IMPL *session, void *huffman_arg, valid += 8; from_bytes--; } - pattern = valid >= max ? /* short patterns near end */ - (bits >> (valid - max)) : (bits << (max - valid)); + pattern = (uint16_t) + (valid >= max ? /* short patterns near end */ + (bits >> (valid - max)) : (bits << (max - valid))); symbol = huffman->code2symbol[pattern & mask]; len = huffman->codes[symbol].length; - valid -= len; + valid -= (uint8_t)len; /* * from_len_bits is the total number of input bits, reduced by diff --git a/src/support/scratch.c b/src/support/scratch.c index 1881f8ad5a5..69987ebc852 100644 --- a/src/support/scratch.c +++ b/src/support/scratch.c @@ -135,6 +135,64 @@ __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) } /* + * __wt_buf_set_printable -- + * Set the contents of the buffer to a printable representation of a + * byte string. + */ +const char * +__wt_buf_set_printable( + WT_SESSION_IMPL *session, const void *p, size_t size, WT_ITEM *buf) +{ + if (__wt_raw_to_esc_hex(session, p, size, buf)) { + buf->data = "[Error]"; + buf->size = strlen("[Error]"); + } + return (buf->data); +} + +/* + * __wt_buf_set_size -- + * Set the contents of the buffer to a printable representation of a + * byte size. + */ +const char * +__wt_buf_set_size( + WT_SESSION_IMPL *session, uint64_t size, bool exact, WT_ITEM *buf) +{ + WT_DECL_RET; + + if (size >= WT_EXABYTE) + ret = __wt_buf_fmt(session, buf, + "%" PRIu64 "EB", size / WT_EXABYTE); + else if (size >= WT_PETABYTE) + ret = __wt_buf_fmt(session, buf, + "%" PRIu64 "PB", size / WT_PETABYTE); + else if (size >= WT_TERABYTE) + ret = __wt_buf_fmt(session, buf, + "%" PRIu64 "TB", size / WT_TERABYTE); + else if (size >= WT_GIGABYTE) + ret = __wt_buf_fmt(session, buf, + "%" PRIu64 "GB", size / WT_GIGABYTE); + else if (size >= WT_MEGABYTE) + ret = __wt_buf_fmt(session, buf, + "%" PRIu64 "MB", size / WT_MEGABYTE); + else if (size >= WT_KILOBYTE) + ret = __wt_buf_fmt(session, buf, + "%" PRIu64 "KB", size / WT_KILOBYTE); + else + ret = __wt_buf_fmt(session, buf, "%" PRIu64 "B", size); + + if (ret == 0 && exact && size >= WT_KILOBYTE) + ret = __wt_buf_catfmt(session, buf, " (%" PRIu64 ")", size); + + if (ret != 0) { + buf->data = "[Error]"; + buf->size = strlen("[Error]"); + } + return (buf->data); +} + +/* * __wt_scr_alloc_func -- * Scratch buffer allocation function. */ diff --git a/src/support/stat.c b/src/support/stat.c index 7514aac56c4..d972f0c140f 100644 --- a/src/support/stat.c +++ b/src/support/stat.c @@ -561,6 +561,8 @@ static const char * const __stats_connection_desc[] = { "cache: eviction server unable to reach eviction goal", "cache: eviction worker thread evicting pages", "cache: failed eviction of pages that exceeded the in-memory maximum", + "cache: files with active eviction walks", + "cache: files with new eviction walks started", "cache: hazard pointer blocked page eviction", "cache: hazard pointer check calls", "cache: hazard pointer check entries walked", @@ -575,15 +577,19 @@ static const char * const __stats_connection_desc[] = { "cache: maximum bytes configured", "cache: maximum page size at eviction", "cache: modified pages evicted", + "cache: modified pages evicted by application threads", "cache: page split during eviction deepened the tree", "cache: page written requiring lookaside records", "cache: pages currently held in the cache", "cache: pages evicted because they exceeded the in-memory maximum", "cache: pages evicted because they had chains of deleted items", "cache: pages evicted by application threads", + "cache: pages queued for eviction", + "cache: pages queued for urgent eviction", "cache: pages read into cache", "cache: pages read into cache requiring lookaside entries", "cache: pages requested from the cache", + "cache: pages seen by eviction walk", "cache: pages selected for eviction unable to be evicted", "cache: pages walked for eviction", "cache: pages written from cache", @@ -604,6 +610,7 @@ static const char * const __stats_connection_desc[] = { "connection: pthread mutex condition wait calls", "connection: pthread mutex shared lock read-lock calls", "connection: pthread mutex shared lock write-lock calls", + "connection: total fsync I/Os", "connection: total read I/Os", "connection: total write I/Os", "cursor: cursor create calls", @@ -646,7 +653,9 @@ static const char * const __stats_connection_desc[] = { "log: log server thread advances write LSN", "log: log server thread write LSN walk skipped", "log: log sync operations", + "log: log sync time duration (usecs)", "log: log sync_dir operations", + "log: log sync_dir time duration (usecs)", "log: log write operations", "log: logging bytes consolidated", "log: maximum log file size", @@ -668,6 +677,9 @@ static const char * const __stats_connection_desc[] = { "reconciliation: split objects currently awaiting free", "session: open cursor count", "session: open session count", + "thread-state: active filesystem fsync calls", + "thread-state: active filesystem read calls", + "thread-state: active filesystem write calls", "thread-yield: page acquire busy blocked", "thread-yield: page acquire eviction blocked", "thread-yield: page acquire locked blocked", @@ -684,6 +696,10 @@ static const char * const __stats_connection_desc[] = { "transaction: transaction checkpoint total time (msecs)", "transaction: transaction checkpoints", "transaction: transaction failures due to cache overflow", + "transaction: transaction fsync calls for checkpoint after allocating the transaction ID", + "transaction: transaction fsync calls for checkpoint before allocating the transaction ID", + "transaction: transaction fsync duration for checkpoint after allocating the transaction ID (usecs)", + "transaction: transaction fsync duration for checkpoint before allocating the transaction ID (usecs)", "transaction: transaction range of IDs currently pinned", "transaction: transaction range of IDs currently pinned by a checkpoint", "transaction: transaction range of IDs currently pinned by named snapshots", @@ -767,6 +783,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_slow = 0; stats->cache_eviction_worker_evicting = 0; stats->cache_eviction_force_fail = 0; + /* not clearing cache_eviction_walks_active */ + stats->cache_eviction_walks_started = 0; stats->cache_eviction_hazard = 0; stats->cache_hazard_checks = 0; stats->cache_hazard_walks = 0; @@ -781,15 +799,19 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing cache_bytes_max */ /* not clearing cache_eviction_maximum_page_size */ stats->cache_eviction_dirty = 0; + stats->cache_eviction_app_dirty = 0; stats->cache_eviction_deepen = 0; stats->cache_write_lookaside = 0; /* not clearing cache_pages_inuse */ stats->cache_eviction_force = 0; stats->cache_eviction_force_delete = 0; stats->cache_eviction_app = 0; + stats->cache_eviction_pages_queued = 0; + stats->cache_eviction_pages_queued_oldest = 0; stats->cache_read = 0; stats->cache_read_lookaside = 0; stats->cache_pages_requested = 0; + stats->cache_eviction_pages_seen = 0; stats->cache_eviction_fail = 0; stats->cache_eviction_walk = 0; stats->cache_write = 0; @@ -810,6 +832,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cond_wait = 0; stats->rwlock_read = 0; stats->rwlock_write = 0; + stats->fsync_io = 0; stats->read_io = 0; stats->write_io = 0; stats->cursor_create = 0; @@ -852,7 +875,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->log_write_lsn = 0; stats->log_write_lsn_skip = 0; stats->log_sync = 0; + stats->log_sync_duration = 0; stats->log_sync_dir = 0; + stats->log_sync_dir_duration = 0; stats->log_writes = 0; stats->log_slot_consolidated = 0; /* not clearing log_max_filesize */ @@ -874,6 +899,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing rec_split_stashed_objects */ /* not clearing session_cursor_open */ /* not clearing session_open */ + /* not clearing fsync_active */ + /* not clearing read_active */ + /* not clearing write_active */ stats->page_busy_blocked = 0; stats->page_forcible_evict_blocked = 0; stats->page_locked_blocked = 0; @@ -890,6 +918,10 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing txn_checkpoint_time_total */ stats->txn_checkpoint = 0; stats->txn_fail_cache = 0; + stats->txn_checkpoint_fsync_post = 0; + stats->txn_checkpoint_fsync_pre = 0; + stats->txn_checkpoint_fsync_post_duration = 0; + stats->txn_checkpoint_fsync_pre_duration = 0; /* not clearing txn_pinned_range */ /* not clearing txn_pinned_checkpoint_range */ /* not clearing txn_pinned_snapshot_range */ @@ -978,6 +1010,10 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cache_eviction_worker_evicting); to->cache_eviction_force_fail += WT_STAT_READ(from, cache_eviction_force_fail); + to->cache_eviction_walks_active += + WT_STAT_READ(from, cache_eviction_walks_active); + to->cache_eviction_walks_started += + WT_STAT_READ(from, cache_eviction_walks_started); to->cache_eviction_hazard += WT_STAT_READ(from, cache_eviction_hazard); to->cache_hazard_checks += WT_STAT_READ(from, cache_hazard_checks); @@ -1001,6 +1037,8 @@ __wt_stat_connection_aggregate( to->cache_eviction_maximum_page_size += WT_STAT_READ(from, cache_eviction_maximum_page_size); to->cache_eviction_dirty += WT_STAT_READ(from, cache_eviction_dirty); + to->cache_eviction_app_dirty += + WT_STAT_READ(from, cache_eviction_app_dirty); to->cache_eviction_deepen += WT_STAT_READ(from, cache_eviction_deepen); to->cache_write_lookaside += @@ -1010,10 +1048,16 @@ __wt_stat_connection_aggregate( to->cache_eviction_force_delete += WT_STAT_READ(from, cache_eviction_force_delete); to->cache_eviction_app += WT_STAT_READ(from, cache_eviction_app); + to->cache_eviction_pages_queued += + WT_STAT_READ(from, cache_eviction_pages_queued); + to->cache_eviction_pages_queued_oldest += + WT_STAT_READ(from, cache_eviction_pages_queued_oldest); to->cache_read += WT_STAT_READ(from, cache_read); to->cache_read_lookaside += WT_STAT_READ(from, cache_read_lookaside); to->cache_pages_requested += WT_STAT_READ(from, cache_pages_requested); + to->cache_eviction_pages_seen += + WT_STAT_READ(from, cache_eviction_pages_seen); to->cache_eviction_fail += WT_STAT_READ(from, cache_eviction_fail); to->cache_eviction_walk += WT_STAT_READ(from, cache_eviction_walk); to->cache_write += WT_STAT_READ(from, cache_write); @@ -1034,6 +1078,7 @@ __wt_stat_connection_aggregate( to->cond_wait += WT_STAT_READ(from, cond_wait); to->rwlock_read += WT_STAT_READ(from, rwlock_read); to->rwlock_write += WT_STAT_READ(from, rwlock_write); + to->fsync_io += WT_STAT_READ(from, fsync_io); to->read_io += WT_STAT_READ(from, read_io); to->write_io += WT_STAT_READ(from, write_io); to->cursor_create += WT_STAT_READ(from, cursor_create); @@ -1078,7 +1123,10 @@ __wt_stat_connection_aggregate( to->log_write_lsn += WT_STAT_READ(from, log_write_lsn); to->log_write_lsn_skip += WT_STAT_READ(from, log_write_lsn_skip); to->log_sync += WT_STAT_READ(from, log_sync); + to->log_sync_duration += WT_STAT_READ(from, log_sync_duration); to->log_sync_dir += WT_STAT_READ(from, log_sync_dir); + to->log_sync_dir_duration += + WT_STAT_READ(from, log_sync_dir_duration); to->log_writes += WT_STAT_READ(from, log_writes); to->log_slot_consolidated += WT_STAT_READ(from, log_slot_consolidated); @@ -1103,6 +1151,9 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, rec_split_stashed_objects); to->session_cursor_open += WT_STAT_READ(from, session_cursor_open); to->session_open += WT_STAT_READ(from, session_open); + to->fsync_active += WT_STAT_READ(from, fsync_active); + to->read_active += WT_STAT_READ(from, read_active); + to->write_active += WT_STAT_READ(from, write_active); to->page_busy_blocked += WT_STAT_READ(from, page_busy_blocked); to->page_forcible_evict_blocked += WT_STAT_READ(from, page_forcible_evict_blocked); @@ -1128,6 +1179,14 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, txn_checkpoint_time_total); to->txn_checkpoint += WT_STAT_READ(from, txn_checkpoint); to->txn_fail_cache += WT_STAT_READ(from, txn_fail_cache); + to->txn_checkpoint_fsync_post += + WT_STAT_READ(from, txn_checkpoint_fsync_post); + to->txn_checkpoint_fsync_pre += + WT_STAT_READ(from, txn_checkpoint_fsync_pre); + to->txn_checkpoint_fsync_post_duration += + WT_STAT_READ(from, txn_checkpoint_fsync_post_duration); + to->txn_checkpoint_fsync_pre_duration += + WT_STAT_READ(from, txn_checkpoint_fsync_pre_duration); to->txn_pinned_range += WT_STAT_READ(from, txn_pinned_range); to->txn_pinned_checkpoint_range += WT_STAT_READ(from, txn_pinned_checkpoint_range); @@ -1139,9 +1198,11 @@ __wt_stat_connection_aggregate( } static const char * const __stats_join_desc[] = { - ": accesses", - ": actual count of items", + ": accesses to the main table", ": bloom filter false positives", + ": checks that conditions of membership are satisfied", + ": items inserted into a bloom filter", + ": items iterated", }; int @@ -1161,9 +1222,11 @@ __wt_stat_join_init_single(WT_JOIN_STATS *stats) void __wt_stat_join_clear_single(WT_JOIN_STATS *stats) { - stats->accesses = 0; - stats->actual_count = 0; + stats->main_access = 0; stats->bloom_false_positive = 0; + stats->membership_check = 0; + stats->bloom_insert = 0; + stats->iterated = 0; } void @@ -1179,7 +1242,9 @@ void __wt_stat_join_aggregate( WT_JOIN_STATS **from, WT_JOIN_STATS *to) { - to->accesses += WT_STAT_READ(from, accesses); - to->actual_count += WT_STAT_READ(from, actual_count); + to->main_access += WT_STAT_READ(from, main_access); to->bloom_false_positive += WT_STAT_READ(from, bloom_false_positive); + to->membership_check += WT_STAT_READ(from, membership_check); + to->bloom_insert += WT_STAT_READ(from, bloom_insert); + to->iterated += WT_STAT_READ(from, iterated); } diff --git a/src/txn/txn.c b/src/txn/txn.c index a2ae97fbd20..dd4384d9a9a 100644 --- a/src/txn/txn.c +++ b/src/txn/txn.c @@ -257,7 +257,7 @@ __txn_oldest_scan(WT_SESSION_IMPL *session, * Sweep the running transactions to update the oldest ID required. */ int -__wt_txn_update_oldest(WT_SESSION_IMPL *session, bool force) +__wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -265,9 +265,12 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, bool force) WT_TXN_GLOBAL *txn_global; uint64_t current_id, last_running, oldest_id; uint64_t prev_last_running, prev_oldest_id; + bool strict, wait; conn = S2C(session); txn_global = &conn->txn_global; + strict = LF_ISSET(WT_TXN_OLDEST_STRICT); + wait = LF_ISSET(WT_TXN_OLDEST_WAIT); current_id = last_running = txn_global->current; prev_last_running = txn_global->last_running; @@ -278,11 +281,11 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, bool force) * oldest ID isn't too far behind, avoid scanning. */ if (prev_oldest_id == current_id || - (!force && WT_TXNID_LT(current_id, prev_oldest_id + 100))) + (!strict && WT_TXNID_LT(current_id, prev_oldest_id + 100))) return (0); /* First do a read-only scan. */ - if (force) + if (wait) WT_RET(__wt_readlock(session, txn_global->scan_rwlock)); else if ((ret = __wt_try_readlock(session, txn_global->scan_rwlock)) != 0) @@ -295,13 +298,13 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, bool force) * non-forced updates), give up. */ if ((oldest_id == prev_oldest_id || - (!force && WT_TXNID_LT(oldest_id, prev_oldest_id + 100))) && + (!strict && WT_TXNID_LT(oldest_id, prev_oldest_id + 100))) && ((last_running == prev_last_running) || - (!force && WT_TXNID_LT(last_running, prev_last_running + 100)))) + (!strict && WT_TXNID_LT(last_running, prev_last_running + 100)))) return (0); /* It looks like an update is necessary, wait for exclusive access. */ - if (force) + if (wait) WT_RET(__wt_writelock(session, txn_global->scan_rwlock)); else if ((ret = __wt_try_writelock(session, txn_global->scan_rwlock)) != 0) @@ -343,6 +346,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, bool force) if (WT_TXNID_LT(txn_global->last_running, last_running)) { txn_global->last_running = last_running; +#ifdef HAVE_VERBOSE /* Output a verbose message about long-running transactions, * but only when some progress is being made. */ if (WT_VERBOSE_ISSET(session, WT_VERB_TRANSACTION) && @@ -355,6 +359,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, bool force) oldest_session->lastop, oldest_session->txn.snap_min)); } +#endif } done: WT_TRET(__wt_writeunlock(session, txn_global->scan_rwlock)); diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index c1b435d9897..51d26b9aed6 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -350,6 +350,7 @@ __checkpoint_verbose_track(WT_SESSION_IMPL *session, static int __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) { + struct timespec fsync_start, fsync_stop; struct timespec start, stop, verb_timer; WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -359,6 +360,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_STATE *txn_state; void *saved_meta_next; u_int i; + uint64_t fsync_duration_usecs; bool full, idle, logging, tracking; const char *txn_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_begin_transaction), "isolation=snapshot", NULL }; @@ -404,7 +406,8 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * This is particularly important for compact, so that all dirty pages * can be fully written. */ - WT_ERR(__wt_txn_update_oldest(session, true)); + WT_ERR(__wt_txn_update_oldest( + session, WT_TXN_OLDEST_STRICT | WT_TXN_OLDEST_WAIT)); /* Flush data-sources before we start the checkpoint. */ WT_ERR(__checkpoint_data_source(session, cfg)); @@ -424,7 +427,13 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * completion. Do it after flushing the pages to give the * asynchronous flush as much time as possible before we wait. */ + WT_ERR(__wt_epoch(session, &fsync_start)); WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); + WT_ERR(__wt_epoch(session, &fsync_stop)); + fsync_duration_usecs = WT_TIMEDIFF_US(fsync_stop, fsync_start); + WT_STAT_FAST_CONN_INCR(session, txn_checkpoint_fsync_pre); + WT_STAT_FAST_CONN_INCRV(session, + txn_checkpoint_fsync_pre_duration, fsync_duration_usecs); /* Tell logging that we are about to start a database checkpoint. */ if (full && logging) @@ -523,7 +532,13 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * Checkpoints have to hit disk (it would be reasonable to configure for * lazy checkpoints, but we don't support them yet). */ + WT_ERR(__wt_epoch(session, &fsync_start)); WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); + WT_ERR(__wt_epoch(session, &fsync_stop)); + fsync_duration_usecs = WT_TIMEDIFF_US(fsync_stop, fsync_start); + WT_STAT_FAST_CONN_INCR(session, txn_checkpoint_fsync_post); + WT_STAT_FAST_CONN_INCRV(session, + txn_checkpoint_fsync_post_duration, fsync_duration_usecs); WT_ERR(__checkpoint_verbose_track(session, "sync completed", &verb_timer)); @@ -1284,7 +1299,8 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) * for active readers. */ if (!btree->modified && !bulk) { - WT_RET(__wt_txn_update_oldest(session, true)); + WT_RET(__wt_txn_update_oldest( + session, WT_TXN_OLDEST_STRICT | WT_TXN_OLDEST_WAIT)); return (__wt_txn_visible_all(session, btree->rec_max_txn) ? __wt_cache_op(session, WT_SYNC_DISCARD) : EBUSY); } diff --git a/src/txn/txn_log.c b/src/txn/txn_log.c index da2670fb344..470515244f3 100644 --- a/src/txn/txn_log.c +++ b/src/txn/txn_log.c @@ -156,6 +156,7 @@ err: __wt_logrec_free(session, &logrec); int __wt_txn_log_op(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) { + WT_DECL_RET; WT_ITEM *logrec; WT_TXN *txn; WT_TXN_OP *op; @@ -179,24 +180,25 @@ __wt_txn_log_op(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) switch (op->type) { case WT_TXN_OP_BASIC: - return (__txn_op_log(session, logrec, op, cbt)); + ret = __txn_op_log(session, logrec, op, cbt); + break; case WT_TXN_OP_INMEM: case WT_TXN_OP_REF: /* Nothing to log, we're done. */ - return (0); + break; case WT_TXN_OP_TRUNCATE_COL: - return (__wt_logop_col_truncate_pack(session, logrec, + ret = __wt_logop_col_truncate_pack(session, logrec, op->fileid, - op->u.truncate_col.start, op->u.truncate_col.stop)); + op->u.truncate_col.start, op->u.truncate_col.stop); + break; case WT_TXN_OP_TRUNCATE_ROW: - return (__wt_logop_row_truncate_pack(session, txn->logrec, + ret = __wt_logop_row_truncate_pack(session, txn->logrec, op->fileid, &op->u.truncate_row.start, &op->u.truncate_row.stop, - (uint32_t)op->u.truncate_row.mode)); - WT_ILLEGAL_VALUE(session); + (uint32_t)op->u.truncate_row.mode); + break; } - - /* NOTREACHED */ + return (ret); } /* diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c index 1ea4dba1152..bd004e0f837 100644 --- a/src/txn/txn_recover.c +++ b/src/txn/txn_recover.c @@ -424,6 +424,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) false, WT_SESSION_NO_LOGGING, &session)); r.session = session; + F_SET(conn, WT_CONN_RECOVERING); WT_ERR(__wt_metadata_search(session, WT_METAFILE_URI, &config)); WT_ERR(__recovery_setup_file(&r, WT_METAFILE_URI, config)); WT_ERR(__wt_metadata_cursor_open(session, NULL, &metac)); @@ -566,6 +567,7 @@ err: WT_TRET(__recovery_free(&r)); WT_TRET(__wt_evict_destroy(session)); WT_TRET(session->iface.close(&session->iface, NULL)); + F_CLR(conn, WT_CONN_RECOVERING); return (ret); } diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c index 3314b5ba485..da70aea35be 100644 --- a/src/utilities/util_dump.c +++ b/src/utilities/util_dump.c @@ -552,14 +552,15 @@ dup_json_string(const char *str, char **result) nchars = 0; for (p = str; *p; p++, nchars++) - nchars += __wt_json_unpack_char(*p, NULL, 0, false); + nchars += __wt_json_unpack_char((u_char)*p, NULL, 0, false); q = malloc(nchars + 1); if (q == NULL) return (1); *result = q; left = nchars; for (p = str; *p; p++, nchars++) { - nchars = __wt_json_unpack_char(*p, (u_char *)q, left, false); + nchars = __wt_json_unpack_char((u_char)*p, (u_char *)q, left, + false); left -= nchars; q += nchars; } diff --git a/src/utilities/util_load.c b/src/utilities/util_load.c index a81d06c6866..ac18df80851 100644 --- a/src/utilities/util_load.c +++ b/src/utilities/util_load.c @@ -211,6 +211,8 @@ config_list_free(CONFIG_LIST *clp) free(*entry); free(clp->list); clp->list = NULL; + clp->entry = 0; + clp->max_entry = 0; } /* diff --git a/src/utilities/util_load_json.c b/src/utilities/util_load_json.c index f1f6675e99c..020a4ed9ba9 100644 --- a/src/utilities/util_load_json.c +++ b/src/utilities/util_load_json.c @@ -248,7 +248,7 @@ json_data(WT_SESSION *session, keyformat = cursor->key_format; isrec = strcmp(keyformat, "r") == 0; for (nkeys = 0; *keyformat; keyformat++) - if (!isdigit(*keyformat)) + if (!__wt_isdigit((u_char)*keyformat)) nkeys++; recno = 0; @@ -427,6 +427,9 @@ json_top_level(WT_SESSION *session, JSON_INPUT_STATE *ins, uint32_t flags) flags)) != 0) goto err; config_list_free(&cl); + free(ins->kvraw); + ins->kvraw = NULL; + config_list_free(&cl); break; } else @@ -468,7 +471,7 @@ json_peek(WT_SESSION *session, JSON_INPUT_STATE *ins) if (!ins->peeking) { while (!ins->ateof) { - while (isspace(*ins->p)) + while (__wt_isspace((u_char)*ins->p)) ins->p++; if (*ins->p) break; @@ -544,15 +547,14 @@ json_skip(WT_SESSION *session, JSON_INPUT_STATE *ins, const char **matches) const char *hit; const char **match; - if (ins->kvraw != NULL) - return (1); - + WT_ASSERT((WT_SESSION_IMPL *)session, ins->kvraw == NULL); hit = NULL; while (!ins->ateof) { for (match = matches; *match != NULL; match++) if ((hit = strstr(ins->p, *match)) != NULL) goto out; - if (util_read_line(session, &ins->line, true, &ins->ateof)) { + if (util_read_line(session, &ins->line, true, &ins->ateof) + != 0) { ins->toktype = -1; return (1); } diff --git a/src/utilities/util_misc.c b/src/utilities/util_misc.c index f45f6b339f2..3c4e8d2dfa1 100644 --- a/src/utilities/util_misc.c +++ b/src/utilities/util_misc.c @@ -108,7 +108,7 @@ util_str2recno(WT_SESSION *session, const char *p, uint64_t *recnop) * forth -- none of them are OK with us. Check the string starts with * digit, that turns off the special processing. */ - if (!isdigit(p[0])) + if (!__wt_isdigit((u_char)p[0])) goto format; errno = 0; diff --git a/src/utilities/util_verify.c b/src/utilities/util_verify.c index 2df4fa65f43..82bdd780cd3 100644 --- a/src/utilities/util_verify.c +++ b/src/utilities/util_verify.c @@ -16,10 +16,10 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) WT_DECL_RET; size_t size; int ch; - bool dump_address, dump_blocks, dump_pages, dump_shape; + bool dump_address, dump_blocks, dump_layout, dump_pages; char *config, *dump_offsets, *name; - dump_address = dump_blocks = dump_pages = dump_shape = false; + dump_address = dump_blocks = dump_layout = dump_pages = false; config = dump_offsets = name = NULL; while ((ch = __wt_getopt(progname, argc, argv, "d:")) != EOF) switch (ch) { @@ -28,6 +28,8 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) dump_address = true; else if (strcmp(__wt_optarg, "dump_blocks") == 0) dump_blocks = true; + else if (strcmp(__wt_optarg, "dump_layout") == 0) + dump_layout = true; else if ( WT_PREFIX_MATCH(__wt_optarg, "dump_offsets=")) { if (dump_offsets != NULL) { @@ -40,8 +42,6 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) __wt_optarg + strlen("dump_offsets="); } else if (strcmp(__wt_optarg, "dump_pages") == 0) dump_pages = true; - else if (strcmp(__wt_optarg, "dump_shape") == 0) - dump_shape = true; else return (usage()); break; @@ -60,12 +60,12 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) /* Build the configuration string as necessary. */ if (dump_address || - dump_blocks || dump_offsets != NULL || dump_pages || dump_shape) { + dump_blocks || dump_layout || dump_offsets != NULL || dump_pages) { size = strlen("dump_address,") + strlen("dump_blocks,") + + strlen("dump_layout,") + strlen("dump_pages,") + - strlen("dump_shape,") + strlen("dump_offsets[],") + (dump_offsets == NULL ? 0 : strlen(dump_offsets)) + 20; if ((config = malloc(size)) == NULL) { @@ -76,11 +76,11 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) "%s%s%s%s%s%s%s", dump_address ? "dump_address," : "", dump_blocks ? "dump_blocks," : "", + dump_layout ? "dump_layout," : "", dump_offsets != NULL ? "dump_offsets=[" : "", dump_offsets != NULL ? dump_offsets : "", dump_offsets != NULL ? "]," : "", - dump_pages ? "dump_pages," : "", - dump_shape ? "dump_shape," : ""); + dump_pages ? "dump_pages," : ""); } if ((ret = session->verify(session, name, config)) != 0) { fprintf(stderr, "%s: verify(%s): %s\n", @@ -109,7 +109,7 @@ usage(void) "usage: %s %s " "verify %s\n", progname, usage_prefix, - "[-d dump_address | dump_blocks | " - "dump_offsets=#,# | dump_pages | dump_shape] uri"); + "[-d dump_address | dump_blocks | dump_layout | " + "dump_offsets=#,# | dump_pages] uri"); return (1); } diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c index e9980cd53cb..9a7584f951f 100644 --- a/test/bloom/test_bloom.c +++ b/test/bloom/test_bloom.c @@ -50,7 +50,8 @@ void cleanup(void); void populate_entries(void); void run(void); void setup(void); -void usage(void); +void usage(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); extern char *__wt_optarg; extern int __wt_optind; diff --git a/test/checkpoint/test_checkpoint.c b/test/checkpoint/test_checkpoint.c index 307cfd914bd..6293d36f916 100644 --- a/test/checkpoint/test_checkpoint.c +++ b/test/checkpoint/test_checkpoint.c @@ -32,7 +32,8 @@ GLOBAL g; static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -static void onint(int); +static void onint(int) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static int cleanup(void); static int usage(void); static int wt_connect(const char *); diff --git a/test/csuite/Makefile.am b/test/csuite/Makefile.am index 6058a05431b..f842bc1316f 100644 --- a/test/csuite/Makefile.am +++ b/test/csuite/Makefile.am @@ -13,6 +13,12 @@ noinst_PROGRAMS += test_wt2246_col_append test_wt2535_insert_race_SOURCES = wt2535_insert_race/main.c noinst_PROGRAMS += test_wt2535_insert_race +test_wt2447_join_main_table_SOURCES = wt2447_join_main_table/main.c +noinst_PROGRAMS += test_wt2447_join_main_table + +test_wt2592_join_schema_SOURCES = wt2592_join_schema/main.c +noinst_PROGRAMS += test_wt2592_join_schema + # Run this during a "make check" smoke test. TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) diff --git a/test/csuite/wt2246_col_append/main.c b/test/csuite/wt2246_col_append/main.c index 3ac96677ed0..798970cbb6d 100644 --- a/test/csuite/wt2246_col_append/main.c +++ b/test/csuite/wt2246_col_append/main.c @@ -45,7 +45,7 @@ void (*custom_die)(void) = NULL; /* Needs to be global for signal handling. */ -TEST_OPTS *opts, _opts; +static TEST_OPTS *opts, _opts; static void page_init(uint64_t n) diff --git a/test/csuite/wt2447_join_main_table/main.c b/test/csuite/wt2447_join_main_table/main.c new file mode 100644 index 00000000000..a6f19cb0858 --- /dev/null +++ b/test/csuite/wt2447_join_main_table/main.c @@ -0,0 +1,189 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-2447 + * + * Test case description: This test case is adapted from the submitted test + * program in the JIRA ticket. We create a database of 10,000 entries, with + * every key i having pair of values (i, i). Create indices on both values, + * and establish a join: table.v1 >= 5000 AND table.v2 < 5001. There's a + * Bloom filter on v2. We expect that although we iterate from 5000 to + * 10000, we'll only have accesses to the main table for key 5000, as + * 5001-10000 will generally not be in the Bloom filter. For key 5000, + * we technically have two accesses to the main table - one occurs when we + * see key 5000 is in the Bloom filter, and we need to do a full test, we + * make an access to the projection table:tablename(v2), that's just to get + * the value of v2, which we'll check by comparison to the cursor at 5001. + * That counts as a main table access, and when we see it is satisfied and + * return the complete set of values, we'll access the main table with the + * full projection (that's the second main table access). + * + * Failure mode: Before fixes of WT-2447, we saw lots of accesses to the main + * table. + */ + +void (*custom_die)(void) = NULL; + +#define N_RECORDS 10000 + +static void +get_stat_total(WT_SESSION *session, WT_CURSOR *jcursor, const char *descmatch, + uint64_t *pval) +{ + WT_CURSOR *statcursor; + uint64_t val; + int ret; + bool match; + char *desc, *valstr; + + match = false; + *pval = 0; + testutil_check(session->open_cursor(session, "statistics:join", jcursor, + NULL, &statcursor)); + + while ((ret = statcursor->next(statcursor)) == 0) { + testutil_assert(statcursor->get_value( + statcursor, &desc, &valstr, &val) == 0); + + printf("statistics: %s: %s: %" PRIu64 "\n", desc, valstr, val); + + if (strstr(desc, descmatch) != NULL) { + *pval += val; + match = true; + } + } + testutil_assert(ret == WT_NOTFOUND); + testutil_check(statcursor->close(statcursor)); + testutil_assert(match); +} + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor1, *cursor2, *jcursor; + WT_ITEM d; + WT_SESSION *session; + uint64_t maincount; + int half, i, j; + const char *tablename; + char bloom_cfg[128], index1uri[256], index2uri[256], joinuri[256]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + tablename = strchr(opts->uri, ':'); + testutil_assert(tablename != NULL); + tablename++; + snprintf(index1uri, sizeof(index1uri), "index:%s:index1", tablename); + snprintf(index2uri, sizeof(index2uri), "index:%s:index2", tablename); + snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + + testutil_check(wiredtiger_open(opts->home, NULL, + "statistics=(all),create", &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->create(session, opts->uri, + "key_format=i,value_format=iiu,columns=(k,v1,v2,d)")); + testutil_check(session->create(session, index1uri, "columns=(v1)")); + testutil_check(session->create(session, index2uri, "columns=(v2)")); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, + &cursor1)); + + d.size = 4100; + d.data = dmalloc(d.size); + memset((char *)d.data, 7, d.size); + + for (i = 0; i < N_RECORDS; ++i) + { + cursor1->set_key(cursor1, i); + cursor1->set_value(cursor1, i, i, &d); + testutil_check(cursor1->insert(cursor1)); + } + + free((void*)d.data); + + testutil_check(opts->conn->close(opts->conn, NULL)); + testutil_check(wiredtiger_open(opts->home, NULL, + "statistics=(all),create,cache_size=1GB", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, + &session)); + + testutil_check(session->open_cursor(session, index1uri, NULL, NULL, + &cursor1)); + testutil_check(session->open_cursor(session, index2uri, NULL, NULL, + &cursor2)); + + half = N_RECORDS / 2; + cursor1->set_key(cursor1, half); + testutil_check(cursor1->search(cursor1)); + + cursor2->set_key(cursor2, half + 1); + testutil_check(cursor2->search(cursor2)); + + sprintf(bloom_cfg, "compare=lt,strategy=bloom,count=%d", half); + + testutil_check(session->open_cursor(session, joinuri, NULL, NULL, + &jcursor)); + testutil_check(session->join(session, jcursor, cursor1, "compare=ge")); + testutil_check(session->join(session, jcursor, cursor2, bloom_cfg)); + + /* Expect one value returned */ + testutil_assert(jcursor->next(jcursor) == 0); + i = 0; + testutil_assert(jcursor->get_key(jcursor, &i) == 0); + testutil_assert(i == (int)half); + i = j = 0; + memset(&d, 0, sizeof(d)); + testutil_assert(jcursor->get_value(jcursor, &i, &j, &d) == 0); + testutil_assert(i == (int)half); + testutil_assert(j == (int)half); + testutil_assert(d.size == 4100); + for (i = 0; i < 4100; i++) + testutil_assert(((char *)d.data)[i] == 7); + + testutil_assert(jcursor->next(jcursor) == WT_NOTFOUND); + + /* + * Make sure there have been 2 accesses to the main table, + * explained in the discussion above. + */ + get_stat_total(session, jcursor, "accesses to the main table", + &maincount); + testutil_assert(maincount == 2); + + testutil_cleanup(opts); + + return (0); +} diff --git a/test/csuite/wt2592_join_schema/main.c b/test/csuite/wt2592_join_schema/main.c new file mode 100644 index 00000000000..4ffc9194646 --- /dev/null +++ b/test/csuite/wt2592_join_schema/main.c @@ -0,0 +1,222 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-2592 + * Test case description: This is an adaptation of the join parts of + * ex_schema.c, but written as a test. Though we have join tests in the + * Python test suite, the Python API uses raw mode for cursors, so errors + * that are specific to non-raw mode are undetected in Python. + * Failure mode: The failure seen in WT-2592 was that no items were returned + * by a join. + */ +#include <inttypes.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <wiredtiger.h> + +/* The C struct for the data we are storing in a WiredTiger table. */ +typedef struct { + char country[5]; + uint16_t year; + uint64_t population; +} POP_RECORD; + +static POP_RECORD pop_data[] = { + { "AU", 1900, 4000000 }, + { "AU", 1950, 8267337 }, + { "AU", 2000, 19053186 }, + { "CAN", 1900, 5500000 }, + { "CAN", 1950, 14011422 }, + { "CAN", 2000, 31099561 }, + { "UK", 1900, 369000000 }, + { "UK", 1950, 50127000 }, + { "UK", 2000, 59522468 }, + { "USA", 1900, 76212168 }, + { "USA", 1950, 150697361 }, + { "USA", 2000, 301279593 }, + { "", 0, 0 } +}; + +void (*custom_die)(void) = NULL; + +int +main(int argc, char *argv[]) +{ + POP_RECORD *p; + TEST_OPTS *opts, _opts; + WT_CURSOR *country_cursor, *country_cursor2, *cursor, *join_cursor, + *subjoin_cursor, *year_cursor; + WT_SESSION *session; + const char *country, *tablename; + char countryuri[256], joinuri[256], yearuri[256]; + uint64_t recno, population; + uint16_t year; + int count, ret; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + tablename = strchr(opts->uri, ':'); + testutil_assert(tablename != NULL); + tablename++; + snprintf(countryuri, sizeof(countryuri), "index:%s:country", tablename); + snprintf(yearuri, sizeof(yearuri), "index:%s:year", tablename); + snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + + testutil_check(wiredtiger_open(opts->home, NULL, + "create,cache_size=200M", &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, + "key_format=r," + "value_format=5sHQ," + "columns=(id,country,year,population)")); + + /* Create an index with a simple key. */ + testutil_check(session->create(session, + countryuri, "columns=(country)")); + + /* Create an immutable index. */ + testutil_check(session->create(session, + yearuri, "columns=(year),immutable")); + + /* Insert the records into the table. */ + testutil_check(session->open_cursor( + session, opts->uri, NULL, "append", &cursor)); + count = 1; + for (p = pop_data; p->year != 0; p++) { + cursor->set_key(cursor, count); + cursor->set_value(cursor, p->country, p->year, p->population); + testutil_check(cursor->insert(cursor)); + count++; + } + testutil_check(cursor->close(cursor)); + + /* Open cursors needed by the join. */ + testutil_check(session->open_cursor(session, + joinuri, NULL, NULL, &join_cursor)); + testutil_check(session->open_cursor(session, + countryuri, NULL, NULL, &country_cursor)); + testutil_check(session->open_cursor(session, + yearuri, NULL, NULL, &year_cursor)); + + /* select values WHERE country == "AU" AND year > 1900 */ + country_cursor->set_key(country_cursor, "AU\0\0\0"); + testutil_check(country_cursor->search(country_cursor)); + testutil_check(session->join(session, join_cursor, country_cursor, + "compare=eq,count=10")); + year_cursor->set_key(year_cursor, (uint16_t)1900); + testutil_check(year_cursor->search(year_cursor)); + testutil_check(session->join(session, join_cursor, year_cursor, + "compare=gt,count=10,strategy=bloom")); + + count = 0; + /* List the values that are joined */ + while ((ret = join_cursor->next(join_cursor)) == 0) { + testutil_check(join_cursor->get_key(join_cursor, &recno)); + testutil_check(join_cursor->get_value(join_cursor, &country, + &year, &population)); + printf("ID %" PRIu64, recno); + printf( + ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", + country, year, population); + count++; + } + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(count == 2); + + testutil_check(join_cursor->close(join_cursor)); + testutil_check(year_cursor->close(year_cursor)); + testutil_check(country_cursor->close(country_cursor)); + + /* Open cursors needed by the join. */ + testutil_check(session->open_cursor(session, + joinuri, NULL, NULL, &join_cursor)); + testutil_check(session->open_cursor(session, + joinuri, NULL, NULL, &subjoin_cursor)); + testutil_check(session->open_cursor(session, + countryuri, NULL, NULL, &country_cursor)); + testutil_check(session->open_cursor(session, + countryuri, NULL, NULL, &country_cursor2)); + testutil_check(session->open_cursor(session, + yearuri, NULL, NULL, &year_cursor)); + + /* + * select values WHERE (country == "AU" OR country == "UK") + * AND year > 1900 + * + * First, set up the join representing the country clause. + */ + country_cursor->set_key(country_cursor, "AU\0\0\0"); + testutil_check(country_cursor->search(country_cursor)); + testutil_check(session->join(session, subjoin_cursor, country_cursor, + "operation=or,compare=eq,count=10")); + country_cursor2->set_key(country_cursor2, "UK\0\0\0"); + testutil_check(country_cursor2->search(country_cursor2)); + testutil_check(session->join(session, subjoin_cursor, country_cursor2, + "operation=or,compare=eq,count=10")); + + /* Join that to the top join, and add the year clause */ + testutil_check(session->join(session, join_cursor, subjoin_cursor, + NULL)); + year_cursor->set_key(year_cursor, (uint16_t)1900); + testutil_check(year_cursor->search(year_cursor)); + testutil_check(session->join(session, join_cursor, year_cursor, + "compare=gt,count=10,strategy=bloom")); + + count = 0; + /* List the values that are joined */ + while ((ret = join_cursor->next(join_cursor)) == 0) { + testutil_check(join_cursor->get_key(join_cursor, &recno)); + testutil_check(join_cursor->get_value(join_cursor, &country, + &year, &population)); + printf("ID %" PRIu64, recno); + printf( + ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", + country, year, population); + count++; + } + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(count == 4); + + testutil_check(join_cursor->close(join_cursor)); + testutil_check(subjoin_cursor->close(subjoin_cursor)); + testutil_check(country_cursor->close(country_cursor)); + testutil_check(country_cursor2->close(country_cursor2)); + testutil_check(year_cursor->close(year_cursor)); + testutil_check(session->close(session, NULL)); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/test/cursor_order/cursor_order.c b/test/cursor_order/cursor_order.c index d8cfc0c1421..aa351e6fea8 100644 --- a/test/cursor_order/cursor_order.c +++ b/test/cursor_order/cursor_order.c @@ -34,7 +34,8 @@ static FILE *logfp; /* Log file */ static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -static void onint(int); +static void onint(int) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void shutdown(void); static int usage(void); static void wt_connect(SHARED_CONFIG *, char *); diff --git a/test/fops/t.c b/test/fops/t.c index 24994404c7c..bf0588d5a53 100644 --- a/test/fops/t.c +++ b/test/fops/t.c @@ -41,7 +41,8 @@ static char home[512]; static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -static void onint(int); +static void onint(int) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void shutdown(void); static int usage(void); static void wt_startup(char *); diff --git a/test/format/bdb.c b/test/format/bdb.c index 48229cfd5e7..e56281f2c3e 100644 --- a/test/format/bdb.c +++ b/test/format/bdb.c @@ -106,9 +106,9 @@ bdb_insert( DBC *dbc; key.data = (void *)key_data; - key.size = (uint32_t)key_size; + key.size = (u_int32_t)key_size; value.data = (void *)value_data; - value.size = (uint32_t)value_size; + value.size = (u_int32_t)value_size; dbc = g.dbc; @@ -147,7 +147,7 @@ bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp) key_gen(&keyitem, keyno); key.data = (void *)keyitem.data; - key.size = keyitem.size; + key.size = (u_int32_t)keyitem.size; *notfoundp = 0; if ((ret = dbc->get(dbc, &key, &value, DB_SET)) != 0) { @@ -169,9 +169,9 @@ bdb_update(const void *arg_key, size_t arg_key_size, int ret; key.data = (void *)arg_key; - key.size = (uint32_t)arg_key_size; + key.size = (u_int32_t)arg_key_size; value.data = (void *)arg_value; - value.size = (uint32_t)arg_value_size; + value.size = (u_int32_t)arg_value_size; if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0) testutil_die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", @@ -188,10 +188,10 @@ bdb_remove(uint64_t keyno, int *notfoundp) key_gen(&keyitem, keyno); key.data = (void *)keyitem.data; - key.size = keyitem.size; + key.size = (u_int32_t)keyitem.size; bdb_read(keyno, &value.data, &size, notfoundp); - value.size = (uint32_t)size; + value.size = (u_int32_t)size; if (*notfoundp) return; diff --git a/test/format/format.h b/test/format/format.h index 8fd9b113311..ad5f408ac30 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -33,12 +33,6 @@ #include <db.h> #endif -#if defined(__GNUC__) -#define WT_GCC_ATTRIBUTE(x) __attribute__(x) -#else -#define WT_GCC_ATTRIBUTE(x) -#endif - #define EXTPATH "../../ext/" /* Extensions path */ #define LZ4_PATH \ @@ -266,7 +260,7 @@ typedef struct { #define TINFO_COMPLETE 2 /* Finished */ #define TINFO_JOINED 3 /* Resolved */ volatile int state; /* state */ -} TINFO WT_GCC_ATTRIBUTE((aligned(WT_CACHE_LINE_ALIGNMENT))); +} TINFO WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT); #ifdef HAVE_BERKELEY_DB void bdb_close(void); diff --git a/test/format/ops.c b/test/format/ops.c index 7fd5563f486..c97d82809a1 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -1465,7 +1465,7 @@ print_item(const char *tag, WT_ITEM *item) static const char hex[] = "0123456789abcdef"; const uint8_t *data; size_t size; - int ch; + u_char ch; data = item->data; size = item->size; @@ -1476,8 +1476,8 @@ print_item(const char *tag, WT_ITEM *item) else for (; size > 0; --size, ++data) { ch = data[0]; - if (isprint(ch)) - fprintf(stderr, "%c", ch); + if (__wt_isprint(ch)) + fprintf(stderr, "%c", (int)ch); else fprintf(stderr, "%x%x", hex[(data[0] & 0xf0) >> 4], diff --git a/test/format/t.c b/test/format/t.c index 085163befe2..2eb2b078804 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -32,7 +32,8 @@ GLOBAL g; static void format_die(void); static void startup(void); -static void usage(void); +static void usage(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); extern int __wt_optind; extern char *__wt_optarg; diff --git a/test/format/wts.c b/test/format/wts.c index 2ee01aa75b5..69195abc3d4 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -87,10 +87,10 @@ handle_message(WT_EVENT_HANDLER *handler, /* Write and flush the message so we're up-to-date on error. */ if (g.logfp == NULL) { - out = printf("%p:%s\n", session, message); + out = printf("%p:%s\n", (void *)session, message); (void)fflush(stdout); } else { - out = fprintf(g.logfp, "%p:%s\n", session, message); + out = fprintf(g.logfp, "%p:%s\n", (void *)session, message); (void)fflush(g.logfp); } return (out < 0 ? EIO : 0); diff --git a/test/huge/huge.c b/test/huge/huge.c index 1e104a705f2..3aa61a9048e 100644 --- a/test/huge/huge.c +++ b/test/huge/huge.c @@ -65,13 +65,13 @@ static size_t lengths[] = { 0 }; +static void usage(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { fprintf(stderr, "usage: %s [-s]\n", progname); - fprintf(stderr, "%s", - "\t-s small run, only test up to 1GB\n"); - + fprintf(stderr, "%s", "\t-s small run, only test up to 1GB\n"); exit(EXIT_FAILURE); } diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index d9639198c34..e485e73067f 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -55,6 +55,8 @@ static const char * const uri = "table:main"; #define MAX_KV 100 #define MAX_VAL 128 +static void usage(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { @@ -68,10 +70,10 @@ extern char *__wt_optarg; void (*custom_die)(void) = NULL; -WT_CONNECTION **connections = NULL; -WT_CURSOR **cursors = NULL; -WT_RAND_STATE rnd; -WT_SESSION **sessions = NULL; +static WT_CONNECTION **connections = NULL; +static WT_CURSOR **cursors = NULL; +static WT_RAND_STATE rnd; +static WT_SESSION **sessions = NULL; static int get_stat(WT_SESSION *stat_session, int stat_field, uint64_t *valuep) diff --git a/test/mciproject.yml b/test/mciproject.yml index 9abdf23ec3b..3df1ce5805e 100644 --- a/test/mciproject.yml +++ b/test/mciproject.yml @@ -8,6 +8,14 @@ functions: command: git.get_project params: directory: wiredtiger + "fetch artifacts" : &fetch_artifacts + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${build_id}.tgz + bucket: build_external + extract_to: wiredtiger pre: - command: shell.exec @@ -21,7 +29,9 @@ post: rm -rf "wiredtiger" tasks: - - name: compile-posix +## Base compile task on posix flavours + - name: compile + depends_on: [] commands: - func: "fetch source" - command: git.apply_patch @@ -33,36 +43,49 @@ tasks: script: | set -o errexit set -o verbose - - ./build_posix/reconf - ${configure_env_vars|} ./configure --enable-diagnostic --enable-python --enable-zlib - ${make_command|make} ${smp_command|} - ${make_command|make} check - - ${test_env_vars|} python ./test/suite/run.py -v 2 - - name: compile-windows - commands: - - func: "fetch source" - - command: git.apply_patch + if [ "Windows_NT" = "$OS" ]; then + scons.bat --enable-python=c:\\swigwin-3.0.2\\swig.exe --enable-diagnostic --enable-verbose ${smp_command|} + else + ./build_posix/reconf + ${configure_env_vars|} ./configure --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-verbose + ${make_command|make} ${smp_command|} 2>&1 + ${make_command|make} check 2>&1 + fi + - command: archive.targz_pack params: - directory: wiredtiger + target: "wiredtiger.tgz" + source_dir: "wiredtiger" + include: + - "./**" + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_file: wiredtiger.tgz + bucket: build_external + permissions: public-read + content_type: application/tar + display_name: Artifacts + remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${build_id}.tgz + + - name: unit-test + depends_on: + - name: compile + commands: + - func: "fetch artifacts" - command: shell.exec params: working_dir: "wiredtiger" script: | set -o errexit set -o verbose - - scons.bat --enable-python=c:\\swigwin-3.0.2\\swig.exe ${smp_command|} - - ${test_env_vars|} python ./test/suite/run.py -v 2 + ${test_env_vars|} python ./test/suite/run.py -v 2 ${smp_command|} 2>&1 - name: compile-windows-alt + depends_on: + - name: compile commands: - - func: "fetch source" - - command: git.apply_patch - params: - directory: wiredtiger + - func: "fetch artifacts" - command: shell.exec params: working_dir: "wiredtiger" @@ -72,22 +95,22 @@ tasks: scons.bat ${smp_command|} "CFLAGS=/Gv /wd4090 /wd4996 /we4047 /we4024 /TC /we4100" wiredtiger.dll libwiredtiger.lib - - name: fops-windows + - name: fops + depends_on: + - name: compile commands: - - func: "fetch source" - - command: git.apply_patch - params: - directory: wiredtiger + - func: "fetch artifacts" - command: shell.exec params: working_dir: "wiredtiger" script: | set -o errexit set -o verbose - - scons.bat --enable-python=c:\\swigwin-3.0.2\\swig.exe ${smp_command|} - - cmd.exe /c t_fops.exe + if [ "Windows_NT" = "$OS" ]; then + cmd.exe /c t_fops.exe + else + ./test/fops/t + fi buildvariants: - name: ubuntu1404 @@ -95,11 +118,14 @@ buildvariants: run_on: - ubuntu1404-test expansions: - test_env_vars: LD_LIBRARY_PATH=.libs - smp_command: -j$(grep -c ^processor /proc/cpuinfo) + # It's ugly, but we need the absolute path here, not the relative + test_env_vars: LD_LIBRARY_PATH=`pwd`/.libs + smp_command: -j $(grep -c ^processor /proc/cpuinfo) configure_env_vars: CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ tasks: - - name: compile-posix + - name: compile + - name: unit-test + - name: fops - name: solaris display_name: Solaris @@ -107,31 +133,34 @@ buildvariants: - solaris expansions: make_command: PATH=/opt/mongodbtoolchain/bin:$PATH gmake - test_env_vars: LD_LIBRARY_PATH=.libs - smp_command: -j$(kstat cpu | sort -u | grep -c "^module") + test_env_vars: LD_LIBRARY_PATH=`pwd`/.libs + smp_command: -j $(kstat cpu | sort -u | grep -c "^module") configure_env_vars: PATH=/opt/mongodbtoolchain/bin:$PATH CFLAGS="-m64" tasks: - - name: compile-posix + - name: compile + - name: unit-test + - name: fops - name: windows-64 display_name: Windows 64-bit run_on: - windows-64-vs2013-test - expansions: - smp_command: -j$(grep -c ^processor /proc/cpuinfo) tasks: - - name: compile-windows + - name: compile - name: compile-windows-alt - - name: fops-windows + - name: unit-test + - name: fops - name: osx-1010 display_name: OS X 10.10 run_on: - osx-1010 expansions: - smp_command: -j$(sysctl -n hw.logicalcpu) + smp_command: -j $(sysctl -n hw.logicalcpu) configure_env_vars: PATH=/opt/local/bin:$PATH make_command: PATH=/opt/local/bin:$PATH ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future make - test_env_vars: DYLD_LIBRARY_PATH=.libs + test_env_vars: DYLD_LIBRARY_PATH=`pwd`/.libs tasks: - - name: compile-posix + - name: compile + - name: unit-test + - name: fops diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 402b99d7d29..31edc0d2a24 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -57,6 +57,8 @@ static const char * const uri = "table:main"; #define OP_READ 0 #define OP_WRITE 1 +static void usage(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { @@ -119,6 +121,9 @@ run_child(const char *homedir, int op, int expect) * Child process opens both databases readonly. */ static void +open_dbs(int, const char *, const char *, + const char *, const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void open_dbs(int op, const char *dir, const char *dir_wr, const char *dir_rd, const char *dir_rd2) { diff --git a/test/recovery/random-abort.c b/test/recovery/random-abort.c index 33597245966..85629eddec4 100644 --- a/test/recovery/random-abort.c +++ b/test/recovery/random-abort.c @@ -35,7 +35,11 @@ static char home[512]; /* Program working dir */ static const char *progname; /* Program name */ static const char * const uri = "table:main"; -#define RECORDS_FILE "records" +#define MAX_TH 12 +#define MIN_TH 5 +#define MAX_TIME 40 +#define MIN_TIME 10 +#define RECORDS_FILE "records-%" PRIu32 #define ENV_CONFIG \ "create,log=(file_max=10M,archive=false,enabled)," \ @@ -43,74 +47,67 @@ static const char * const uri = "table:main"; #define ENV_CONFIG_REC "log=(recover=on)" #define MAX_VAL 4096 +static void usage(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, "usage: %s [-h dir]\n", progname); + fprintf(stderr, "usage: %s [-h dir] [-T threads]\n", progname); exit(EXIT_FAILURE); } -/* - * Child process creates the database and table, and then writes data into - * the table until it is killed by the parent. - */ -static void -fill_db(void) +typedef struct { + WT_CONNECTION *conn; + uint64_t start; + uint32_t id; +} WT_THREAD_DATA; + +static void * +thread_run(void *arg) { FILE *fp; - WT_CONNECTION *conn; WT_CURSOR *cursor; WT_ITEM data; WT_RAND_STATE rnd; WT_SESSION *session; + WT_THREAD_DATA *td; uint64_t i; int ret; - uint8_t buf[MAX_VAL]; + char buf[MAX_VAL], kname[64]; __wt_random_init(&rnd); memset(buf, 0, sizeof(buf)); - /* - * Initialize the first 25% to random values. Leave a bunch of data - * space at the end to emphasize zero data. - */ - for (i = 0; i < MAX_VAL/4; i++) - buf[i] = (uint8_t)__wt_random(&rnd); + memset(kname, 0, sizeof(kname)); + td = (WT_THREAD_DATA *)arg; /* - * Run in the home directory so that the records file is in there too. + * The value is the name of the record file with our id appended. */ - if (chdir(home) != 0) - testutil_die(errno, "chdir: %s", home); - if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) - testutil_die(ret, "wiredtiger_open"); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - testutil_die(ret, "WT_CONNECTION:open_session"); - if ((ret = session->create(session, - uri, "key_format=Q,value_format=u")) != 0) - testutil_die(ret, "WT_SESSION.create: %s", uri); - if ((ret = - session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) - testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); - + snprintf(buf, sizeof(buf), RECORDS_FILE, td->id); /* * Keep a separate file with the records we wrote for checking. */ - (void)unlink(RECORDS_FILE); - if ((fp = fopen(RECORDS_FILE, "w")) == NULL) + (void)unlink(buf); + if ((fp = fopen(buf, "w")) == NULL) testutil_die(errno, "fopen"); /* * Set to no buffering. */ - __wt_stream_set_no_buffer(fp); - + __wt_stream_set_line_buffer(fp); + if ((ret = td->conn->open_session(td->conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "WT_CONNECTION:open_session"); + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); + data.data = buf; + data.size = sizeof(buf); /* - * Write data into the table until we are killed by the parent. - * The data in the buffer is already set to random content. + * Write our portion of the key space until we're killed. */ - data.data = buf; - for (i = 0;; ++i) { + for (i = td->start; ; ++i) { + snprintf(kname, sizeof(kname), "%" PRIu64, i); data.size = __wt_random(&rnd) % MAX_VAL; - cursor->set_key(cursor, i); + cursor->set_key(cursor, kname); cursor->set_value(cursor, &data); if ((ret = cursor->insert(cursor)) != 0) testutil_die(ret, "WT_CURSOR.insert"); @@ -119,9 +116,63 @@ fill_db(void) */ if (fprintf(fp, "%" PRIu64 "\n", i) == -1) testutil_die(errno, "fprintf"); - if (i % 5000) - __wt_yield(); } + /* NOTREACHED */ +} + +/* + * Child process creates the database and table, and then creates worker + * threads to add data until it is killed by the parent. + */ +static void fill_db(uint32_t) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void +fill_db(uint32_t nth) +{ + pthread_t *thr; + WT_CONNECTION *conn; + WT_SESSION *session; + WT_THREAD_DATA *td; + uint32_t i; + int ret; + + thr = dcalloc(nth, sizeof(pthread_t)); + td = dcalloc(nth, sizeof(WT_THREAD_DATA)); + if (chdir(home) != 0) + testutil_die(errno, "Child chdir: %s", home); + if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) + testutil_die(ret, "wiredtiger_open"); + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "WT_CONNECTION:open_session"); + if ((ret = session->create(session, + uri, "key_format=S,value_format=u")) != 0) + testutil_die(ret, "WT_SESSION.create: %s", uri); + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "WT_SESSION:close"); + + printf("Create %" PRIu32 " writer threads\n", nth); + for (i = 0; i < nth; ++i) { + td[i].conn = conn; + td[i].start = (UINT64_MAX / nth) * i; + td[i].id = i; + if ((ret = pthread_create( + &thr[i], NULL, thread_run, &td[i])) != 0) + testutil_die(ret, "pthread_create"); + } + printf("Spawned %" PRIu32 " writer threads\n", nth); + fflush(stdout); + /* + * The threads never exit, so the child will just wait here until + * it is killed. + */ + for (i = 0; i < nth; ++i) + testutil_assert(pthread_join(thr[i], NULL) == 0); + /* + * NOTREACHED + */ + free(thr); + free(td); + exit(EXIT_SUCCESS); } extern int __wt_optind; @@ -138,26 +189,40 @@ main(int argc, char *argv[]) WT_SESSION *session; WT_RAND_STATE rnd; uint64_t key; - uint32_t absent, count, timeout; + uint32_t absent, count, i, nth, timeout; int ch, status, ret; pid_t pid; + bool rand_th, rand_time, verify_only; const char *working_dir; + char fname[64], kname[64]; if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) progname = argv[0]; else ++progname; + nth = MIN_TH; + rand_th = rand_time = true; + timeout = MIN_TIME; + verify_only = false; working_dir = "WT_TEST.random-abort"; - timeout = 10; - while ((ch = __wt_getopt(progname, argc, argv, "h:t:")) != EOF) + + while ((ch = __wt_getopt(progname, argc, argv, "h:T:t:v")) != EOF) switch (ch) { case 'h': working_dir = __wt_optarg; break; + case 'T': + rand_th = false; + nth = (uint32_t)atoi(__wt_optarg); + break; case 't': + rand_time = false; timeout = (uint32_t)atoi(__wt_optarg); break; + case 'v': + verify_only = true; + break; default: usage(); } @@ -167,43 +232,68 @@ main(int argc, char *argv[]) usage(); testutil_work_dir_from_path(home, 512, working_dir); - testutil_make_work_dir(home); - /* - * Fork a child to insert as many items. We will then randomly - * kill the child, run recovery and make sure all items we wrote - * exist after recovery runs. + * If the user wants to verify they need to tell us how many threads + * there were so we can find the old record files. */ - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); - - if (pid == 0) { /* child */ - fill_db(); - return (EXIT_SUCCESS); + if (verify_only && rand_th) { + fprintf(stderr, + "Verify option requires specifying number of threads\n"); + exit (EXIT_FAILURE); } + if (!verify_only) { + testutil_make_work_dir(home); - /* parent */ - __wt_random_init(&rnd); - /* Sleep for the configured amount of time before killing the child. */ - printf("Parent: sleep %" PRIu32 " seconds, then kill child\n", timeout); - sleep(timeout); + testutil_assert(__wt_random_init_seed(NULL, &rnd) == 0); + if (rand_time) { + timeout = __wt_random(&rnd) % MAX_TIME; + if (timeout < MIN_TIME) + timeout = MIN_TIME; + } + if (rand_th) { + nth = __wt_random(&rnd) % MAX_TH; + if (nth < MIN_TH) + nth = MIN_TH; + } + printf("Parent: Create %" PRIu32 + " threads; sleep %" PRIu32 " seconds\n", nth, timeout); + /* + * Fork a child to insert as many items. We will then randomly + * kill the child, run recovery and make sure all items we wrote + * exist after recovery runs. + */ + if ((pid = fork()) < 0) + testutil_die(errno, "fork"); - /* - * !!! It should be plenty long enough to make sure more than one - * log file exists. If wanted, that check would be added here. - */ - printf("Kill child\n"); - if (kill(pid, SIGKILL) != 0) - testutil_die(errno, "kill"); - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); + if (pid == 0) { /* child */ + fill_db(nth); + return (EXIT_SUCCESS); + } + /* parent */ + /* + * Sleep for the configured amount of time before killing + * the child. + */ + sleep(timeout); + + /* + * !!! It should be plenty long enough to make sure more than + * one log file exists. If wanted, that check would be added + * here. + */ + printf("Kill child\n"); + if (kill(pid, SIGKILL) != 0) + testutil_die(errno, "kill"); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); + } /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. */ if (chdir(home) != 0) - testutil_die(errno, "chdir: %s", home); + testutil_die(errno, "parent chdir: %s", home); printf("Open database, run recovery and verify content\n"); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); @@ -213,30 +303,39 @@ main(int argc, char *argv[]) session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); - if ((fp = fopen(RECORDS_FILE, "r")) == NULL) - testutil_die(errno, "fopen"); + absent = count = 0; + for (i = 0; i < nth; ++i) { + snprintf(fname, sizeof(fname), RECORDS_FILE, i); + if ((fp = fopen(fname, "r")) == NULL) { + fprintf(stderr, + "Failed to open %s. i %" PRIu32 "\n", fname, i); + testutil_die(errno, "fopen"); + } - /* - * For every key in the saved file, verify that the key exists - * in the table after recovery. Since we did write-no-sync, we - * expect every key to have been recovered. - */ - for (absent = count = 0;; ++count) { - ret = fscanf(fp, "%" SCNu64 "\n", &key); - if (ret != EOF && ret != 1) - testutil_die(errno, "fscanf"); - if (ret == EOF) - break; - cursor->set_key(cursor, key); - if ((ret = cursor->search(cursor)) != 0) { - if (ret != WT_NOTFOUND) - testutil_die(ret, "search"); - printf("no record with key %" PRIu64 "\n", key); - ++absent; + /* + * For every key in the saved file, verify that the key exists + * in the table after recovery. Since we did write-no-sync, we + * expect every key to have been recovered. + */ + for (;; ++count) { + ret = fscanf(fp, "%" SCNu64 "\n", &key); + if (ret != EOF && ret != 1) + testutil_die(errno, "fscanf"); + if (ret == EOF) + break; + snprintf(kname, sizeof(kname), "%" PRIu64, key); + cursor->set_key(cursor, kname); + if ((ret = cursor->search(cursor)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + printf("%s: no record with key %" PRIu64 "\n", + fname, key); + ++absent; + } } + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); } - if (fclose(fp) != 0) - testutil_die(errno, "fclose"); if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (absent) { diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c index 3b99ea2c932..a7509c27566 100644 --- a/test/recovery/truncated-log.c +++ b/test/recovery/truncated-log.c @@ -50,6 +50,8 @@ static const char * const uri = "table:main"; #define K_SIZE 16 #define V_SIZE 256 +static void usage(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { @@ -61,6 +63,7 @@ usage(void) * Child process creates the database and table, and then writes data into * the table until it is killed by the parent. */ +static void fill_db(void)WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void fill_db(void) { diff --git a/test/salvage/salvage.c b/test/salvage/salvage.c index 497902e07b8..c3349188623 100644 --- a/test/salvage/salvage.c +++ b/test/salvage/salvage.c @@ -701,7 +701,7 @@ print_res(int key, int value, int cnt) switch (page_type) { /* Print value */ case WT_PAGE_COL_FIX: ch = value & 0x7f; - if (isprint(ch)) { + if (__wt_isprint((u_char)ch)) { if (ch == '\\') fputc('\\', res_fp); fputc(ch, res_fp); diff --git a/test/suite/run.py b/test/suite/run.py index f7f0d1399ff..6e7421b8b96 100644 --- a/test/suite/run.py +++ b/test/suite/run.py @@ -51,7 +51,7 @@ elif os.path.isfile(os.path.join(wt_disttop, 'wt.exe')): wt_builddir = wt_disttop else: print 'Unable to find useable WiredTiger build' - sys.exit(False) + sys.exit(1) # Cannot import wiredtiger and supporting utils until we set up paths # We want our local tree in front of any installed versions of WiredTiger. @@ -241,7 +241,7 @@ if __name__ == '__main__': if option == '-dir' or option == 'D': if dirarg != None or len(args) == 0: usage() - sys.exit(False) + sys.exit(2) dirarg = args.pop(0) continue if option == '-debug' or option == 'd': @@ -252,14 +252,14 @@ if __name__ == '__main__': continue if option == '-help' or option == 'h': usage() - sys.exit(True) + sys.exit(0) if option == '-long' or option == 'l': longtest = True continue if option == '-parallel' or option == 'j': if parallel != 0 or len(args) == 0: usage() - sys.exit(False) + sys.exit(2) parallel = int(args.pop(0)) continue if option == '-preserve' or option == 'p': @@ -271,7 +271,7 @@ if __name__ == '__main__': if option == '-verbose' or option == 'v': if len(args) == 0: usage() - sys.exit(False) + sys.exit(2) verbose = int(args.pop(0)) if verbose > 3: verbose = 3 @@ -281,19 +281,19 @@ if __name__ == '__main__': if option == '-config' or option == 'c': if configfile != None or len(args) == 0: usage() - sys.exit(False) + sys.exit(2) configfile = args.pop(0) continue if option == '-configcreate' or option == 'C': if configfile != None or len(args) == 0: usage() - sys.exit(False) + sys.exit(2) configfile = args.pop(0) configwrite = True continue print 'unknown arg: ' + arg usage() - sys.exit(False) + sys.exit(2) testargs.append(arg) # All global variables should be set before any test classes are loaded. @@ -318,4 +318,4 @@ if __name__ == '__main__': pdb.set_trace() result = wttest.runsuite(tests, parallel) - sys.exit(not result.wasSuccessful()) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/test/suite/suite_subprocess.py b/test/suite/suite_subprocess.py index df89d82e4c9..c56c8d8e933 100644 --- a/test/suite/suite_subprocess.py +++ b/test/suite/suite_subprocess.py @@ -117,13 +117,12 @@ class suite_subprocess: print 'ERROR: ' + filename + ' should not be empty (this command expected error output)' self.assertNotEqual(filesize, 0, filename + ': expected to not be empty') - def runWt(self, args, infilename=None, outfilename=None, errfilename=None, reopensession=True): - """ - Run the 'wt' process - """ + # Run the wt utility. + def runWt(self, args, infilename=None, + outfilename=None, errfilename=None, reopensession=True, failure=False): - # we close the connection to guarantee everything is - # flushed, and that we can open it from another process + # Close the connection to guarantee everything is flushed, and that + # we can open it from another process. self.close_conn() wtoutname = outfilename or "wt.out" @@ -141,14 +140,26 @@ class suite_subprocess: infilepart = "<" + infilename + " " print str(procargs) print "*********************************************" - print "**** Run 'wt' via: run " + " ".join(procargs[3:]) + infilepart + ">" + wtoutname + " 2>" + wterrname + print "**** Run 'wt' via: run " + \ + " ".join(procargs[3:]) + infilepart + \ + ">" + wtoutname + " 2>" + wterrname print "*********************************************" - subprocess.call(procargs) + returncode = subprocess.call(procargs) elif infilename: with open(infilename, "r") as wtin: - subprocess.call(procargs, stdin=wtin, stdout=wtout, stderr=wterr) + returncode = subprocess.call( + procargs, stdin=wtin, stdout=wtout, stderr=wterr) else: - subprocess.call(procargs, stdout=wtout, stderr=wterr) + returncode = subprocess.call( + procargs, stdout=wtout, stderr=wterr) + if failure: + self.assertNotEqual(returncode, 0, + 'expected failure: "' + \ + str(procargs) + '": exited ' + str(returncode)) + else: + self.assertEqual(returncode, 0, + 'expected success: "' + \ + str(procargs) + '": exited ' + str(returncode)) if errfilename == None: self.check_empty_file(wterrname) if outfilename == None: diff --git a/test/suite/test_config04.py b/test/suite/test_config04.py index 7186bc3a716..dffa7479f1b 100644 --- a/test/suite/test_config04.py +++ b/test/suite/test_config04.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os +import os, shutil import wiredtiger, wttest from wiredtiger import stat @@ -34,6 +34,7 @@ from wiredtiger import stat # Individually test config options class test_config04(wttest.WiredTigerTestCase): table_name1 = 'test_config04' + log1 = 'WiredTigerLog.0000000001' nentries = 100 K = 1024 @@ -86,6 +87,10 @@ class test_config04(wttest.WiredTigerTestCase): self.assertEqual(cursor[stat.conn.cache_bytes_max][2], size) cursor.close() + def common_log_test(self, path, dirname): + self.common_test('log=(archive=false,enabled,' + path + ')') + self.assertTrue(os.path.exists(dirname + os.sep + self.log1)) + def test_bad_config(self): msg = '/unknown configuration key/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, @@ -168,24 +173,46 @@ class test_config04(wttest.WiredTigerTestCase): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.wiredtiger_open('.', '(create='), msg) - def test_session_max(self): - # Note: There isn't any direct way to know that this was set, - # but we'll have a separate functionality test to test for - # this indirectly. - self.common_test('session_max=99') - - def test_multiprocess(self): - self.common_test('multiprocess') - # TODO: how do we verify that it was set? - def test_error_prefix(self): self.common_test('error_prefix="MyOwnPrefix"') # TODO: how do we verify that it was set? def test_logging(self): - self.common_test('log=(enabled=true)') - # TODO: how do we verify that it was set? For this we could look - # for the existence of the log file in the home dir. + # Test variations on the log configuration. The log test takes + # a configuration string as the first arg and the directory pathname + # to confirm the existence of the log file. For now we're testing + # the log pathname only. + # + # Test the default in the home directory. + self.common_log_test('', '.') + self.conn.close() + + # Test a subdir of the home directory. + logdirname = 'logdir' + logdir = '.' + os.sep + logdirname + os.mkdir(logdir) + confstr = 'path=' + logdirname + self.common_log_test(confstr, logdir) + self.conn.close() + + # Test an absolute path directory. + if os.name == 'posix': + logdir = '/tmp/logdir' + os.mkdir(logdir) + confstr = 'path=' + logdir + self.common_log_test(confstr, logdir) + self.conn.close() + shutil.rmtree(logdir, ignore_errors=True) + + def test_multiprocess(self): + self.common_test('multiprocess') + # TODO: how do we verify that it was set? + + def test_session_max(self): + # Note: There isn't any direct way to know that this was set, + # but we'll have a separate functionality test to test for + # this indirectly. + self.common_test('session_max=99') def test_transactional(self): # Note: this will have functional tests in the future. diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py index d0163066639..85196174c1b 100644 --- a/test/suite/test_dump.py +++ b/test/suite/test_dump.py @@ -143,8 +143,8 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): self.populate_check(self, uri, self.nentries) # Re-load the object again, but confirm -n (no overwrite) fails. - self.runWt(['-h', self.dir, - 'load', '-n', '-f', 'dump.out'], errfilename='errfile.out') + self.runWt(['-h', self.dir, 'load', '-n', '-f', 'dump.out'], + errfilename='errfile.out', failure=True) self.check_non_empty_file('errfile.out') # If there are indices, dump one of them and check the output. diff --git a/test/suite/test_jsondump02.py b/test/suite/test_jsondump02.py index 251237f3faf..50931f0f5e6 100644 --- a/test/suite/test_jsondump02.py +++ b/test/suite/test_jsondump02.py @@ -208,12 +208,16 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): # this one should work self.load_json(self.table_uri2, - (('"key0" : "KEY002"', '"value0" : 345,\n"value1" : "str2"'),)) + (('"key0" : "KEY002"', '"value0" : 34,\n"value1" : "str2"'),)) # extraneous/missing space is okay self.load_json(self.table_uri2, ((' "key0"\n:\t"KEY003" ', - '"value0":456,"value1"\n\n\r\n:\t\n"str3"'),)) + '"value0":45,"value1"\n\n\r\n:\t\n"str3"'),)) + + table2_json = ( + ('"key0" : "KEY002"', '"value0" : 34,\n"value1" : "str2"'), + ('"key0" : "KEY003"', '"value0" : 45,\n"value1" : "str3"')) table3_json = ( ('"key0" : 1', '"value0" : "\\u0001\\u0002\\u0003"'), @@ -284,12 +288,11 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): self.runWt(['load', '-jf', 'jsondump4.out']) self.session.drop(self.table_uri4) - # Note: only the first table is loaded. self.runWt(['load', '-jf', 'jsondump-all.out']) self.check_json(self.table_uri1, table1_json) - #self.check_json(self.table_uri2, table2_json) - #self.check_json(self.table_uri3, table3_json) - #self.check_json(self.table_uri4, table4_json) + self.check_json(self.table_uri2, table2_json) + self.check_json(self.table_uri3, table3_json) + self.check_json(self.table_uri4, table4_json) # Generate two byte keys that cover some range of byte values. # For simplicity, the keys are monotonically increasing. diff --git a/test/suite/test_util02.py b/test/suite/test_util02.py index 51e03d8d105..475e856052a 100644 --- a/test/suite/test_util02.py +++ b/test/suite/test_util02.py @@ -173,7 +173,7 @@ class test_load_commandline(wttest.WiredTigerTestCase, suite_subprocess): complex_populate(self, self.uri, "key_format=S,value_format=S", 20) self.runWt(["dump", self.uri], outfilename="dump.out") loadargs = ["load", "-f", "dump.out"] + args - self.runWt(loadargs, errfilename=errfile) + self.runWt(loadargs, errfilename=errfile, failure=fail) if fail: self.check_non_empty_file(errfile) else: @@ -181,23 +181,24 @@ class test_load_commandline(wttest.WiredTigerTestCase, suite_subprocess): # Empty arguments should suceed. def test_load_commandline_1(self): - self.load_commandline([], 0) + self.load_commandline([], False) # Arguments are in pairs. def test_load_commandline_2(self): - self.load_commandline(["table"], 1) - self.load_commandline([self.uri, "block_allocation=first", self.uri], 1) + self.load_commandline(["table"], True) + self.load_commandline( + [self.uri, "block_allocation=first", self.uri], True) # You can use short-hand URIs for a single object, but cannot match multiple # objects. def test_load_commandline_3(self): - self.load_commandline(["table", "block_allocation=first"], 0) - self.load_commandline(["colgroup", "block_allocation=first"], 1) + self.load_commandline(["table", "block_allocation=first"], False) + self.load_commandline(["colgroup", "block_allocation=first"], True) # You can't reference non-existent objects. def test_load_commandline_4(self): - self.load_commandline([self.uri, "block_allocation=first"], 0) - self.load_commandline(["table:bar", "block_allocation=first"], 1) + self.load_commandline([self.uri, "block_allocation=first"], False) + self.load_commandline(["table:bar", "block_allocation=first"], True) # You can specify multipleconfiguration arguments for the same object. def test_load_commandline_5(self): @@ -205,19 +206,19 @@ class test_load_commandline(wttest.WiredTigerTestCase, suite_subprocess): self.uri, "block_allocation=first", self.uri, "block_allocation=best", self.uri, "block_allocation=first", - self.uri, "block_allocation=best"], 0) + self.uri, "block_allocation=best"], False) # You can't modify a format. def test_load_commandline_6(self): - self.load_commandline(["table", "key_format=d"], 1) - self.load_commandline(["table", "value_format=d"], 1) + self.load_commandline(["table", "key_format=d"], True) + self.load_commandline(["table", "value_format=d"], True) # You can set the source or version, but it gets stripped; confirm the # attempt succeeds, so we know they configuration values are stripped. def test_load_commandline_7(self): - self.load_commandline(["table", "filename=bar"], 0) - self.load_commandline(["table", "source=bar"], 0) - self.load_commandline(["table", "version=(100,200)"], 0) + self.load_commandline(["table", "filename=bar"], False) + self.load_commandline(["table", "source=bar"], False) + self.load_commandline(["table", "version=(100,200)"], False) if __name__ == '__main__': diff --git a/test/suite/test_util07.py b/test/suite/test_util07.py index 2bbb40422bd..602ddbba5ff 100644 --- a/test/suite/test_util07.py +++ b/test/suite/test_util07.py @@ -71,7 +71,8 @@ class test_util07(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, self.session_params) outfile = "readout.txt" errfile = "readerr.txt" - self.runWt(["read", 'table:' + self.tablename, 'NoMatch'], outfilename=outfile, errfilename=errfile) + self.runWt(["read", 'table:' + self.tablename, 'NoMatch'], + outfilename=outfile, errfilename=errfile, failure=True) self.check_empty_file(outfile) self.check_file_contains(errfile, 'NoMatch: not found\n') @@ -83,10 +84,12 @@ class test_util07(wttest.WiredTigerTestCase, suite_subprocess): self.populate(self.tablename) outfile = "readout.txt" errfile = "readerr.txt" - self.runWt(["read", 'table:' + self.tablename, 'KEY49'], outfilename=outfile, errfilename=errfile) + self.runWt(["read", 'table:' + self.tablename, 'KEY49'], + outfilename=outfile, errfilename=errfile) self.check_file_content(outfile, 'VAL49\n') self.check_empty_file(errfile) - self.runWt(["read", 'table:' + self.tablename, 'key49'], outfilename=outfile, errfilename=errfile) + self.runWt(["read", 'table:' + self.tablename, 'key49'], + outfilename=outfile, errfilename=errfile, failure=True) self.check_empty_file(outfile) self.check_file_contains(errfile, 'key49: not found\n') diff --git a/test/suite/test_util12.py b/test/suite/test_util12.py index e8226a3146c..f407c2ce7d6 100644 --- a/test/suite/test_util12.py +++ b/test/suite/test_util12.py @@ -57,7 +57,8 @@ class test_util12(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, self.session_params) errfile = 'writeerr.txt' - self.runWt(['write', 'table:' + self.tablename], errfilename=errfile) + self.runWt(['write', 'table:' + self.tablename], + errfilename=errfile, failure=True) self.check_file_contains(errfile, 'usage:') def test_write_overwrite(self): @@ -82,7 +83,7 @@ class test_util12(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, self.session_params) errfile = 'writeerr.txt' self.runWt(['write', 'table:' + self.tablename, - 'def', '456', 'abc'], errfilename=errfile) + 'def', '456', 'abc'], errfilename=errfile, failure=True) self.check_file_contains(errfile, 'usage:') diff --git a/test/suite/test_verify.py b/test/suite/test_verify.py index 5ce926027ef..28a66415b9d 100644 --- a/test/suite/test_verify.py +++ b/test/suite/test_verify.py @@ -151,7 +151,8 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): with self.open_and_position(self.tablename, 75) as f: for i in range(0, 4096): f.write(struct.pack('B', 0)) - self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.runWt(["verify", "table:" + self.tablename], + errfilename="verifyerr.out", failure=True) self.check_non_empty_file("verifyerr.out") def test_verify_process_25pct_junk(self): @@ -165,7 +166,8 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): with self.open_and_position(self.tablename, 25) as f: for i in range(0, 100): f.write('\x01\xff\x80') - self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.runWt(["verify", "table:" + self.tablename], + errfilename="verifyerr.out", failure=True) self.check_non_empty_file("verifyerr.out") def test_verify_process_truncated(self): @@ -178,7 +180,8 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.populate(self.tablename) with self.open_and_position(self.tablename, 75) as f: f.truncate(0) - self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.runWt(["verify", "table:" + self.tablename], + errfilename="verifyerr.out", failure=True) self.check_non_empty_file("verifyerr.out") def test_verify_process_zero_length(self): @@ -190,7 +193,8 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.populate(self.tablename) with self.open_and_position(self.tablename, 0) as f: f.truncate(0) - self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.runWt(["verify", "table:" + self.tablename], + errfilename="verifyerr.out", failure=True) self.check_non_empty_file("verifyerr.out") diff --git a/test/suite/wttest.py b/test/suite/wttest.py index f7a2a5c8890..9e430fcdba7 100644 --- a/test/suite/wttest.py +++ b/test/suite/wttest.py @@ -551,4 +551,4 @@ def runsuite(suite, parallel): def run(name='__main__'): result = runsuite(unittest.TestLoader().loadTestsFromName(name), False) - sys.exit(not result.wasSuccessful()) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/test/thread/t.c b/test/thread/t.c index 22334076ee1..5b53532e8a6 100644 --- a/test/thread/t.c +++ b/test/thread/t.c @@ -42,7 +42,8 @@ static FILE *logfp; /* Log file */ static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -static void onint(int); +static void onint(int) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void shutdown(void); static int usage(void); static void wt_connect(char *); diff --git a/test/utility/test_util.h b/test/utility/test_util.h index 6417c5a326b..66ff8de2d19 100644 --- a/test/utility/test_util.h +++ b/test/utility/test_util.h @@ -109,10 +109,7 @@ typedef struct { extern void (*custom_die)(void); void testutil_die(int, const char *, ...) -#if defined(__GNUC__) -__attribute__((__noreturn__)) -#endif -; + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); void *dcalloc(size_t, size_t); void *dmalloc(size_t); diff --git a/tools/wtstats/stat_data.py b/tools/wtstats/stat_data.py index eca3c137738..a79cf1faf5e 100644 --- a/tools/wtstats/stat_data.py +++ b/tools/wtstats/stat_data.py @@ -5,6 +5,7 @@ no_scale_per_second_list = [ 'async: maximum work queue length', 'cache: bytes currently in the cache', 'cache: eviction currently operating in aggressive mode', + 'cache: files with active eviction walks', 'cache: hazard pointer maximum array length', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', @@ -27,6 +28,9 @@ no_scale_per_second_list = [ 'reconciliation: split objects currently awaiting free', 'session: open cursor count', 'session: open session count', + 'thread-state: active filesystem fsync calls', + 'thread-state: active filesystem read calls', + 'thread-state: active filesystem write calls', 'transaction: transaction checkpoint currently running', 'transaction: transaction checkpoint generation', 'transaction: transaction checkpoint max time (msecs)', @@ -72,6 +76,7 @@ no_clear_list = [ 'async: maximum work queue length', 'cache: bytes currently in the cache', 'cache: eviction currently operating in aggressive mode', + 'cache: files with active eviction walks', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', 'cache: pages currently held in the cache', @@ -93,6 +98,9 @@ no_clear_list = [ 'reconciliation: split objects currently awaiting free', 'session: open cursor count', 'session: open session count', + 'thread-state: active filesystem fsync calls', + 'thread-state: active filesystem read calls', + 'thread-state: active filesystem write calls', 'transaction: transaction checkpoint currently running', 'transaction: transaction checkpoint generation', 'transaction: transaction checkpoint max time (msecs)', @@ -119,6 +127,7 @@ prefix_list = [ 'thread-yield', 'async', 'btree', + 'thread-state', 'compression', ] -groups = {'cursor': ['cursor', 'session'], 'lsm': ['LSM', 'transaction'], 'system': ['connection', 'data-handle', 'session'], 'evict': ['cache', 'connection', 'block-manager'], 'memory': ['cache', 'connection', 'reconciliation']}
\ No newline at end of file +groups = {'cursor': ['cursor', 'session'], 'lsm': ['LSM', 'transaction'], 'system': ['connection', 'data-handle', 'session', 'thread-state'], 'evict': ['block-manager', 'cache', 'connection', 'thread-state'], 'memory': ['cache', 'connection', 'reconciliation']}
\ No newline at end of file |