diff options
author | Ramon Fernandez <ramon@mongodb.com> | 2016-09-26 08:11:23 -0400 |
---|---|---|
committer | Ramon Fernandez <ramon@mongodb.com> | 2016-09-26 08:11:23 -0400 |
commit | 9dda827a3ae58beef36d53da1b55554cbd8744c4 (patch) | |
tree | 6bcd26e1daf5f808a1e5b1dce3f75925aae8b773 /src/third_party/wiredtiger | |
parent | 0de201d310304852d97a7fd4f3248110c83f29b5 (diff) | |
download | mongo-9dda827a3ae58beef36d53da1b55554cbd8744c4.tar.gz |
Import wiredtiger: fc0e7abe82595e579573d42448632f7b36a2d154 from branch mongodb-3.4
ref: 5bc03723a7..fc0e7abe82
for: 3.3.15
WT-2864 Reconfiguring the checkpoint server can lead to hangs
WT-2874 Change test_compact01 to avoid eviction
WT-2918 The dist scripts create C files s_whitespace complains about
WT-2919 Don't mask error returns from style checking scripts
WT-2921 Reduce the WT_SESSION hazard_size when possible
WT-2923 heap-use-after-free on address in compaction
WT-2924 Ensure we are doing eviction when threads are waiting for it
WT-2925 WT_THREAD_PANIC_FAIL is a WT_THREAD structure flag
WT-2926 WT_CONNECTION.reconfigure can attempt unlock of not-locked lock
WT-2928 Eviction failing to switch queues can lead to starvation
Diffstat (limited to 'src/third_party/wiredtiger')
33 files changed, 342 insertions, 194 deletions
diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index 33b8f6a76ba..31abab28910 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -56,7 +56,7 @@ errchk() # Some tests shouldn't return an error, we exclude them here. case "$1" in - *s_export*) + *s_export|*s_tags) break;; *) errfound=1;; @@ -124,3 +124,4 @@ echo 'dist/s_all run finished' if test $errmode -ne 0; then exit $errfound; fi +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_c_test_create b/src/third_party/wiredtiger/dist/s_c_test_create index 616570dca2e..f4f9eb3ac1f 100755 --- a/src/third_party/wiredtiger/dist/s_c_test_create +++ b/src/third_party/wiredtiger/dist/s_c_test_create @@ -12,7 +12,7 @@ # script in the main C suite directory. # tmp=__a -trap 'rm -f $tmp; exit 0' 0 1 2 3 13 15 +trap 'rm -f $tmp' 0 1 2 3 13 15 if [ "x$1" = "x" ]; then echo "Usage: $0 test_name" diff --git a/src/third_party/wiredtiger/dist/s_copyright b/src/third_party/wiredtiger/dist/s_copyright index cfada71a839..9ff6c20492e 100755 --- a/src/third_party/wiredtiger/dist/s_copyright +++ b/src/third_party/wiredtiger/dist/s_copyright @@ -53,7 +53,7 @@ if [ $# -ne 0 ]; then exit 0 fi -trap 'rm -f $c1 $c2 $c3 $c4; exit 0' 0 1 2 3 13 15 +trap 'rm -f $c1 $c2 $c3 $c4' 0 1 2 3 13 15 year=`date +%Y` @@ -141,3 +141,5 @@ special_copyright LICENSE 1 "$string1" special_copyright src/docs/build-javadoc.sh 1 "$string2" special_copyright src/docs/style/footer.html 2 "$string2" special_copyright src/utilities/util_cpyright.c 1 "$string3" + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_define b/src/third_party/wiredtiger/dist/s_define index 050101e8510..d5bdee7c45f 100755 --- a/src/third_party/wiredtiger/dist/s_define +++ b/src/third_party/wiredtiger/dist/s_define @@ -2,7 +2,7 @@ # Complain about unused #defines. t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # List of source files to search. l=`sed -e '/^[a-z]/!d' -e 's/[ ].*$//' -e 's,^,../,' filelist` diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 08602989fe8..e2b1d2aed11 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -1,7 +1,7 @@ #! /bin/sh t=__wt.$$ -trap 'rm -f $t /tmp/__doxy; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t /tmp/__doxy' 0 1 2 3 13 15 # Skip this when building release packages: docs are built separately test -n "$WT_RELEASE_BUILD" && exit 0 diff --git a/src/third_party/wiredtiger/dist/s_export b/src/third_party/wiredtiger/dist/s_export index 94242e0118d..dc69238b270 100755 --- a/src/third_party/wiredtiger/dist/s_export +++ b/src/third_party/wiredtiger/dist/s_export @@ -2,8 +2,8 @@ # Check for illegal external symbols. # -t=__a.c -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +t=__wt.$$ +trap 'rm -f $t' 0 1 2 3 13 15 case `uname` in Darwin) @@ -40,8 +40,10 @@ check() } # This check would normally be done after the library is built, but this way -# we don't forget about a symbol during development. Check the previously -# built library, if it exists. +# we don't forget about a symbol during development. We usually build in the +# top-level or build_posix directories, check the previously built library, +# if it exists. And, allow this script to be run from the top-level directory +# as well as locally. for d in .libs build_posix/.libs; do f="$d/libwiredtiger.a" test -f $f && check $f diff --git a/src/third_party/wiredtiger/dist/s_funcs b/src/third_party/wiredtiger/dist/s_funcs index 8695c8d4fa7..70ddc7cbd68 100755 --- a/src/third_party/wiredtiger/dist/s_funcs +++ b/src/third_party/wiredtiger/dist/s_funcs @@ -2,7 +2,7 @@ # Complain about unused functions t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # List of files to search. l=`sed -e '/^[a-z]/!d' -e 's/[ ].*$//' -e 's,^,../,' filelist` diff --git a/src/third_party/wiredtiger/dist/s_label b/src/third_party/wiredtiger/dist/s_label index b56ecc6fc78..c7b63d9d5b3 100755 --- a/src/third_party/wiredtiger/dist/s_label +++ b/src/third_party/wiredtiger/dist/s_label @@ -2,7 +2,7 @@ # Check WiredTiger error/return macros. t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 cd .. @@ -79,3 +79,5 @@ for f in `find bench examples ext src test -name '*.[ci]'`; do sed -e "s,^,$f:," -e 's/$/ [return skips API_END call]/' done + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_lang b/src/third_party/wiredtiger/dist/s_lang index 0f0519f87e4..cd229b4ea29 100755 --- a/src/third_party/wiredtiger/dist/s_lang +++ b/src/third_party/wiredtiger/dist/s_lang @@ -2,7 +2,7 @@ # Check lang directories for potential name conflicts t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 cd ../lang @@ -21,5 +21,8 @@ for d in *; do echo "$l: potential SWIG naming conflict" echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" cat $t + exit 1 } done + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_longlines b/src/third_party/wiredtiger/dist/s_longlines index 43e350022dd..bdb9811b4bd 100755 --- a/src/third_party/wiredtiger/dist/s_longlines +++ b/src/third_party/wiredtiger/dist/s_longlines @@ -2,7 +2,7 @@ # Check for long lines t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 l=`(cd .. && find bench/wtperf examples ext src test -name '*.[chisy]' && @@ -20,3 +20,5 @@ for f in $l ; do expand -t8 < ../$f | awk -- \ "{if(length(\$0) > 80) printf(\"%s:%d\\n\", \"$f\", NR)}" done + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_prototypes b/src/third_party/wiredtiger/dist/s_prototypes index 5633c3b5140..4c07b9aa160 100755 --- a/src/third_party/wiredtiger/dist/s_prototypes +++ b/src/third_party/wiredtiger/dist/s_prototypes @@ -2,7 +2,7 @@ # Build a list of internal function and variable prototypes. t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # proto -- # extract public functions. @@ -83,3 +83,5 @@ l=`sed \ -e '/os_win/d' \ -e 's/[ ].*$//' filelist` externs + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_python b/src/third_party/wiredtiger/dist/s_python index ce955328288..b8aa5848637 100755 --- a/src/third_party/wiredtiger/dist/s_python +++ b/src/third_party/wiredtiger/dist/s_python @@ -2,7 +2,7 @@ # Python style checks. t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 cd .. @@ -25,3 +25,5 @@ test -s $t && { echo 'trailing semi-colons in selected Python code:' cat $t } + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_readme b/src/third_party/wiredtiger/dist/s_readme index be809a6455c..316e60773e0 100755 --- a/src/third_party/wiredtiger/dist/s_readme +++ b/src/third_party/wiredtiger/dist/s_readme @@ -1,7 +1,7 @@ #! /bin/sh t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 f=../README . ../RELEASE_INFO @@ -60,3 +60,5 @@ END_TEXT cmp $t $f > /dev/null 2>&1 || (echo "Building $f" && rm -f $f && cp $t $f) + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_stat b/src/third_party/wiredtiger/dist/s_stat index 0638a7f3337..935c7e1fb43 100755 --- a/src/third_party/wiredtiger/dist/s_stat +++ b/src/third_party/wiredtiger/dist/s_stat @@ -2,7 +2,7 @@ # Complain about unused statistics fields. t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # List of files to search: skip stat.c, it lists all of the fields by # definition. diff --git a/src/third_party/wiredtiger/dist/s_string b/src/third_party/wiredtiger/dist/s_string index 32aa7528979..b7b0a4bbdba 100755 --- a/src/third_party/wiredtiger/dist/s_string +++ b/src/third_party/wiredtiger/dist/s_string @@ -3,7 +3,7 @@ # Check spelling in comments and quoted strings from the source files. t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # Insulate against locale-specific sort order LC_ALL=C diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style index 47f18ef1f18..3860a23b991 100755 --- a/src/third_party/wiredtiger/dist/s_style +++ b/src/third_party/wiredtiger/dist/s_style @@ -2,7 +2,7 @@ # General style correction and cleanup. t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # Parallelize if possible. xp="" @@ -183,3 +183,5 @@ else cmp $t $f > /dev/null 2>&1 || (echo "modifying $f" && cp $t $f) fi + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_tags b/src/third_party/wiredtiger/dist/s_tags index faed132d05b..edb1567992c 100755 --- a/src/third_party/wiredtiger/dist/s_tags +++ b/src/third_party/wiredtiger/dist/s_tags @@ -1,41 +1,43 @@ #! /bin/sh -# Build tags file. -# -t=__a.c -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +# Build tags files. +trap 'rm -f tags' 0 1 2 3 13 15 # Skip this when building release packages test -n "$WT_RELEASE_BUILD" && exit 0 +# We have to be in the dist directory to run. +test -f s_tags || { + echo "s_tags requires dist be the current working directory" + exit 1 +} + # We require ctags which may not be installed. type ctags > /dev/null 2>&1 || { - echo 'skipped: ctags not found' > $t + echo 'skipped: ctags not found' exit 0 } # Test to see what flags this ctags binary supports. -# Use the -d, -t and -w flags to ctags if available. flags="" -echo "f() { int a; }" > $t for i in -d -t -w --language-force=C; do - if ctags $i $t 2>/dev/null; then + if ctags $i ../src/conn/api_strerror.c 2>/dev/null; then flags="$i $flags" fi done -# Generate a tags file for the build directory -(cd ../build_posix -rm -f tags -ctags $flags ../src/include/*.in ../src/*/*.[chi] 2>/dev/null) +# Generate a tags file for the standard build directory. +(cd ../build_posix && +rm -f tags && +ctags $flags ../src/include/*.in `find ../src -name '*.[chi]'` 2>/dev/null) -# Put the shared tags file in the include directory, it's at the same level in -# the tree as the other source files. -(cd ../src/include -rm -f tags -ctags $flags ../include/*.in ../*/*.[chi] 2>/dev/null) +# Generate a tags file for the src/include directory. +(cd ../src/include && +rm -f tags && +ctags $flags ../include/*.in `find .. -name '*.[chi]'` 2>/dev/null) -# Link the tags file into place if we're at the right level. +# Link the tags file into place in the standard source directories, if we're +# at the right level. link_tag() { if test -e ../include/tags; then @@ -46,8 +48,9 @@ link_tag() # Link to the tags file from standard build and source directories. dirs="`python -c 'import dist; dist.print_source_dirs()'` ../src/os_win" for i in $dirs; do - if expr "$i" : ".*/include" > /dev/null; then - continue - fi + expr "$i" : ".*/include" > /dev/null && continue + (cd $i && link_tag) done + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_typedef b/src/third_party/wiredtiger/dist/s_typedef index b044a0e6b4b..d4bcda94e74 100755 --- a/src/third_party/wiredtiger/dist/s_typedef +++ b/src/third_party/wiredtiger/dist/s_typedef @@ -1,7 +1,7 @@ #! /bin/sh t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # Insulate against locale-specific sort order and IFS from the user's env LC_ALL=C diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index ab2f23a5fdd..f7bfbcc7e8e 100644 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -1,7 +1,7 @@ #! /bin/sh t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 cd .. @@ -132,3 +132,5 @@ for f in `find bench ext src test -name '*.[ci]'`; do cat $t } done + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_whitespace b/src/third_party/wiredtiger/dist/s_whitespace index e60d85f8cc6..0de59bc5825 100755 --- a/src/third_party/wiredtiger/dist/s_whitespace +++ b/src/third_party/wiredtiger/dist/s_whitespace @@ -2,7 +2,7 @@ # Single space and remove trailing whitespace from source files. t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # Clear lines that only contain whitespace, compress multiple empty lines # into a single line, discard trailing empty lines. @@ -30,3 +30,5 @@ find bench dist examples ext src test \ | while read f ; do whitespace $f done + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_wtstats b/src/third_party/wiredtiger/dist/s_wtstats index 834b011110e..82eee1ab1cf 100755 --- a/src/third_party/wiredtiger/dist/s_wtstats +++ b/src/third_party/wiredtiger/dist/s_wtstats @@ -3,7 +3,7 @@ # Create wtstats.template.html file t=__wt.$$ -trap 'rm -f $t; exit 0' 0 1 2 3 13 15 +trap 'rm -f $t' 0 1 2 3 13 15 # We require npm which may not be installed. type npm > /dev/null 2>&1 || { diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c index 7ba45f29b76..e005674762d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_compact.c +++ b/src/third_party/wiredtiger/src/btree/bt_compact.c @@ -171,30 +171,64 @@ int __wt_compact_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) { WT_BM *bm; + WT_DECL_RET; size_t addr_size; u_int type; const uint8_t *addr; - *skipp = false; /* Default to reading. */ - type = 0; /* Keep compiler quiet. */ + /* + * Skip deleted pages, rewriting them doesn't seem useful; in a better + * world we'd write the parent to delete the page. + */ + if (ref->state == WT_REF_DELETED) { + *skipp = true; + return (0); + } - bm = S2BT(session)->bm; + *skipp = false; /* Default to reading */ /* - * We aren't holding a hazard pointer, so we can't look at the page - * itself, all we can look at is the WT_REF information. If there's no - * address, the page isn't on disk, but we have to read internal pages - * to walk the tree regardless; throw up our hands and read it. + * If the page is in-memory, we want to look at it (it may have been + * modified and written, and the current location is the interesting + * one in terms of compaction, not the original location). + * + * This test could be combined with the next one, but this is a cheap + * test and the next one is expensive. */ - __wt_ref_info(ref, &addr, &addr_size, &type); - if (addr == NULL) + if (ref->state != WT_REF_DISK) + return (0); + + /* + * There's nothing to prevent the WT_REF state from changing underfoot, + * which can change its address. For example, the WT_REF address might + * reference an on-page cell, and page eviction can free that memory. + * Lock the WT_REF so we can look at its address. + */ + if (!__wt_atomic_casv32(&ref->state, WT_REF_DISK, WT_REF_LOCKED)) return (0); /* + * The page is on disk, so there had better be an address; assert that + * fact, test at run-time to avoid the core dump. + * * Internal pages must be read to walk the tree; ask the block-manager * if it's useful to rewrite leaf pages, don't do the I/O if a rewrite * won't help. */ - return (type == WT_CELL_ADDR_INT ? 0 : - bm->compact_page_skip(bm, session, addr, addr_size, skipp)); + __wt_ref_info(ref, &addr, &addr_size, &type); + WT_ASSERT(session, addr != NULL); + if (addr != NULL && type != WT_CELL_ADDR_INT) { + bm = S2BT(session)->bm; + ret = bm->compact_page_skip( + bm, session, addr, addr_size, skipp); + } + + /* + * Reset the WT_REF state and push the change. The full-barrier isn't + * necessary, but it's better to keep pages in circulation than not. + */ + ref->state = WT_REF_DISK; + WT_FULL_BARRIER(); + + return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index dc1cf4e7f98..fb0d2296823 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -472,27 +472,14 @@ restart: /* empty_internal = false; } else if (LF_ISSET(WT_READ_COMPACT)) { /* - * Skip deleted pages, rewriting them doesn't - * seem useful. + * Compaction has relatively complex tests to + * decide if a page can be skipped, call out + * to a helper function. */ - if (ref->state == WT_REF_DELETED) + WT_ERR(__wt_compact_page_skip( + session, ref, &skip)); + if (skip) break; - - /* - * If the page is in-memory, we want to look at - * it (it may have been modified and written, - * and the current location is the interesting - * one in terms of compaction, not the original - * location). If the page isn't in-memory, test - * if the page will help with compaction, don't - * read it if we don't have to. - */ - if (ref->state == WT_REF_DISK) { - WT_ERR(__wt_compact_page_skip( - session, ref, &skip)); - if (skip) - break; - } } else { /* * Try to skip deleted pages visible to us. diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index faec72a4ac0..0951fd4e58c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1055,13 +1055,16 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) WT_DECL_RET; WT_SESSION_IMPL *session; const char *p; + bool locked; conn = (WT_CONNECTION_IMPL *)wt_conn; + locked = false; CONNECTION_API_CALL(conn, session, reconfigure, config, cfg); /* Serialize reconfiguration. */ __wt_spin_lock(session, &conn->reconfig_lock); + locked = true; /* * The configuration argument has been checked for validity, update the @@ -1096,7 +1099,8 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) __wt_free(session, conn->cfg); conn->cfg = p; -err: __wt_spin_unlock(session, &conn->reconfig_lock); +err: if (locked) + __wt_spin_unlock(session, &conn->reconfig_lock); API_END_RET(session, ret); } @@ -1117,11 +1121,11 @@ __conn_open_session(WT_CONNECTION *wt_conn, *wt_sessionp = NULL; conn = (WT_CONNECTION_IMPL *)wt_conn; - session_ret = NULL; CONNECTION_API_CALL(conn, session, open_session, config, cfg); WT_UNUSED(cfg); + session_ret = NULL; WT_ERR(__wt_open_session( conn, event_handler, config, true, &session_ret)); *wt_sessionp = &session_ret->iface; diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 35c057c9767..acc81f566a5 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -152,12 +152,24 @@ __wt_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref) /* * __evict_queue_empty -- * Is the queue empty? + * + * Note that the eviction server is pessimistic and treats a half full + * queue as empty. */ static inline bool -__evict_queue_empty(WT_EVICT_QUEUE *queue) +__evict_queue_empty(WT_EVICT_QUEUE *queue, bool server_check) { - return (queue->evict_current == NULL || - queue->evict_candidates == 0); + uint32_t candidates, used; + + if (queue->evict_current == NULL) + return (true); + + /* The eviction server only considers half of the candidates. */ + candidates = queue->evict_candidates; + if (server_check && candidates > 1) + candidates /= 2; + used = (uint32_t)(queue->evict_current - queue->evict_queue); + return (used >= candidates); } /* @@ -431,7 +443,6 @@ __evict_update_work(WT_SESSION_IMPL *session) { WT_CACHE *cache; WT_CONNECTION_IMPL *conn; - double dirty_trigger; uint64_t bytes_inuse, bytes_max, dirty_inuse; conn = S2C(session); @@ -443,7 +454,7 @@ __evict_update_work(WT_SESSION_IMPL *session) if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) return (false); - if (!__evict_queue_empty(cache->evict_urgent_queue)) + if (!__evict_queue_empty(cache->evict_urgent_queue, false)) F_SET(cache, WT_CACHE_EVICT_URGENT); /* @@ -456,16 +467,14 @@ __evict_update_work(WT_SESSION_IMPL *session) bytes_inuse = __wt_cache_bytes_inuse(cache); if (bytes_inuse > (cache->eviction_target * bytes_max) / 100) F_SET(cache, WT_CACHE_EVICT_CLEAN); - if (bytes_inuse > (cache->eviction_trigger * bytes_max) / 100) - F_SET(cache, WT_CACHE_EVICT_CLEAN_HARD); + if (__wt_eviction_clean_needed(session, NULL)) + F_SET(cache, WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_CLEAN_HARD); dirty_inuse = __wt_cache_dirty_leaf_inuse(cache); if (dirty_inuse > (cache->eviction_dirty_target * bytes_max) / 100) F_SET(cache, WT_CACHE_EVICT_DIRTY); - if ((dirty_trigger = cache->eviction_scrub_limit) < 1.0) - dirty_trigger = (double)cache->eviction_dirty_trigger; - if (dirty_inuse > (uint64_t)(dirty_trigger * bytes_max) / 100) - F_SET(cache, WT_CACHE_EVICT_DIRTY_HARD); + if (__wt_eviction_dirty_needed(session, NULL)) + F_SET(cache, WT_CACHE_EVICT_DIRTY | WT_CACHE_EVICT_DIRTY_HARD); /* * If application threads are blocked by the total volume of data in @@ -497,6 +506,12 @@ __evict_update_work(WT_SESSION_IMPL *session) F_CLR(cache, WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_CLEAN_HARD); } + /* If threads are blocked by eviction we should be looking for pages. */ + WT_ASSERT(session, !F_ISSET(cache, WT_CACHE_EVICT_CLEAN_HARD) || + F_ISSET(cache, WT_CACHE_EVICT_CLEAN)); + WT_ASSERT(session, !F_ISSET(cache, WT_CACHE_EVICT_DIRTY_HARD) || + F_ISSET(cache, WT_CACHE_EVICT_DIRTY)); + WT_STAT_CONN_SET(session, cache_eviction_state, F_MASK(cache, WT_CACHE_EVICT_MASK)); @@ -585,7 +600,7 @@ __evict_pass(WT_SESSION_IMPL *session) */ if (cache->evict_empty_score < WT_EVICT_SCORE_CUTOFF || (!WT_EVICT_HAS_WORKERS(session) && - !__evict_queue_empty(cache->evict_urgent_queue))) + !__evict_queue_empty(cache->evict_urgent_queue, false))) WT_RET(__evict_lru_pages(session, true)); if (cache->pass_intr != 0) @@ -921,7 +936,7 @@ __evict_lru_walk(WT_SESSION_IMPL *session) * If the queue we are filling is empty, pages are being requested * faster than they are being queued. */ - if (__evict_queue_empty(queue)) { + if (__evict_queue_empty(queue, false)) { if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN_HARD | WT_CACHE_EVICT_DIRTY_HARD)) { cache->evict_empty_score = WT_MIN( @@ -1530,7 +1545,7 @@ __evict_get_ref( WT_CACHE *cache; WT_DECL_RET; WT_EVICT_ENTRY *evict; - WT_EVICT_QUEUE *other_queue, *queue, *urgent_queue; + WT_EVICT_QUEUE *queue, *other_queue, *urgent_queue; uint32_t candidates; bool is_app, urgent_ok; @@ -1546,9 +1561,9 @@ __evict_get_ref( WT_STAT_CONN_INCR(session, cache_eviction_get_ref); /* Avoid the LRU lock if no pages are available. */ - if (__evict_queue_empty(cache->evict_current_queue) && - __evict_queue_empty(cache->evict_other_queue) && - __evict_queue_empty(urgent_queue)) { + if (__evict_queue_empty(cache->evict_current_queue, is_server) && + __evict_queue_empty(cache->evict_other_queue, is_server) && + (!urgent_ok || __evict_queue_empty(urgent_queue, false))) { WT_STAT_CONN_INCR(session, cache_eviction_get_ref_empty); return (WT_NOTFOUND); } @@ -1562,54 +1577,51 @@ __evict_get_ref( * Such cases are extremely rare in real applications. */ if (is_server && - (cache->evict_empty_score > WT_EVICT_SCORE_CUTOFF || - __evict_queue_empty(cache->evict_fill_queue))) { - do { + (cache->evict_empty_score > WT_EVICT_SCORE_CUTOFF || + __evict_queue_empty(cache->evict_fill_queue, false))) { + while ((ret = __wt_spin_trylock( + session, &cache->evict_queue_lock)) == EBUSY) if ((!urgent_ok || - __evict_queue_empty(urgent_queue)) && + __evict_queue_empty(urgent_queue, false)) && !__evict_queue_full(cache->evict_fill_queue)) return (WT_NOTFOUND); - } while ((ret = __wt_spin_trylock( - session, &cache->evict_queue_lock)) == EBUSY); WT_RET(ret); } else __wt_spin_lock(session, &cache->evict_queue_lock); - /* - * Check if the current queue needs to change. - * The current queue could have changed while we waited for the lock. - */ - queue = cache->evict_current_queue; - other_queue = cache->evict_other_queue; - if (__evict_queue_empty(queue) && !__evict_queue_empty(other_queue)) { - cache->evict_current_queue = other_queue; - cache->evict_other_queue = queue; - } - /* Check the urgent queue first. */ - queue = urgent_ok && !__evict_queue_empty(urgent_queue) ? - urgent_queue : cache->evict_current_queue; + if (urgent_ok && !__evict_queue_empty(urgent_queue, false)) + queue = urgent_queue; + else { + /* + * Check if the current queue needs to change. + * The current queue could have changed while we waited for + * the lock. + * + * The server will only evict half of the pages before looking + * for more. The remainder are left to eviction workers (if any + * configured), or application threads if necessary. + */ + queue = cache->evict_current_queue; + other_queue = cache->evict_other_queue; + if (__evict_queue_empty(queue, is_server) && + !__evict_queue_empty(other_queue, is_server)) { + cache->evict_current_queue = other_queue; + cache->evict_other_queue = queue; + } + } __wt_spin_unlock(session, &cache->evict_queue_lock); /* - * Only evict half of the pages before looking for more. The remainder - * are left to eviction workers (if configured), or application threads - * if necessary. - */ - candidates = queue->evict_candidates; - if (is_server && queue != urgent_queue && candidates > 1) - candidates /= 2; - - /* * We got the queue lock, which should be fast, and chose a queue. * Now we want to get the lock on the individual queue. */ for (;;) { /* Verify there are still pages available. */ - if (__evict_queue_empty(queue) || (uint32_t) - (queue->evict_current - queue->evict_queue) >= candidates) { + if (__evict_queue_empty( + queue, is_server && queue != urgent_queue)) { WT_STAT_CONN_INCR( session, cache_eviction_get_ref_empty2); return (WT_NOTFOUND); @@ -1621,6 +1633,15 @@ __evict_get_ref( break; } + /* + * Only evict half of the pages before looking for more. The remainder + * are left to eviction workers (if configured), or application thread + * if necessary. + */ + candidates = queue->evict_candidates; + if (is_server && queue != urgent_queue && candidates > 1) + candidates /= 2; + /* Get the next page queued for eviction. */ for (evict = queue->evict_current; evict >= queue->evict_queue && @@ -1676,8 +1697,8 @@ __evict_get_ref( } /* Move to the next item. */ - if (evict != NULL && evict + 1 < - queue->evict_queue + queue->evict_candidates) + if (evict != NULL && + evict + 1 < queue->evict_queue + queue->evict_candidates) queue->evict_current = evict + 1; else /* Clear the current pointer if there are no more candidates. */ queue->evict_current = NULL; @@ -1771,22 +1792,9 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) /* Wake the eviction server if we need to do work. */ __wt_evict_server_wake(session); - /* - * If we're busy, either because of the transaction check we just did, - * or because our caller is waiting on a longer-than-usual event (such - * as a page read), limit the work to a single eviction and return. If - * that's not the case, we can do more. - */ init_evict_count = cache->pages_evict; for (;;) { - /* Check if we have become busy. */ - if (!busy && txn_state->snap_min != WT_TXN_NONE && - txn_global->current != txn_global->oldest_id) - busy = true; - - max_pages_evicted = busy ? 5 : 20; - /* * A pathological case: if we're the oldest transaction in the * system and the eviction server is stuck trying to find space, @@ -1799,6 +1807,20 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) return (WT_ROLLBACK); } + /* + * Check if we have become busy. + * + * If we're busy (because of the transaction check we just did + * or because our caller is waiting on a longer-than-usual event + * such as a page read), and the cache level drops below 100%, + * limit the work to 5 evictions and return. If that's not the + * case, we can do more. + */ + if (!busy && txn_state->snap_min != WT_TXN_NONE && + txn_global->current != txn_global->oldest_id) + busy = true; + max_pages_evicted = busy ? 5 : 20; + /* See if eviction is still needed. */ if (!__wt_eviction_needed(session, busy, &pct_full) || (pct_full < 100 && @@ -1869,7 +1891,7 @@ __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref) goto done; __wt_spin_lock(session, &urgent_queue->evict_lock); - if (__evict_queue_empty(urgent_queue)) { + if (__evict_queue_empty(urgent_queue, false)) { urgent_queue->evict_current = urgent_queue->evict_queue; urgent_queue->evict_candidates = 0; } diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 6e32c1bc195..a9ce4f754a9 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -406,7 +406,7 @@ __wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page) /* Update pages and bytes evicted. */ (void)__wt_atomic_add64(&cache->bytes_evict, page->memory_footprint); - (void)__wt_atomic_add64(&cache->pages_evict, 1); + (void)__wt_atomic_addv64(&cache->pages_evict, 1); } /* diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 515135f26ab..b24b625aec4 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -66,7 +66,7 @@ struct __wt_cache { uint64_t bytes_dirty_leaf; uint64_t pages_dirty_leaf; uint64_t bytes_evict; /* Bytes/pages discarded by eviction */ - uint64_t pages_evict; + volatile uint64_t pages_evict; uint64_t pages_evicted; /* Pages evicted during a pass */ uint64_t bytes_image; /* Bytes of disk images */ uint64_t bytes_inmem; /* Bytes/pages in memory */ @@ -175,7 +175,7 @@ struct __wt_cache { #define WT_CACHE_EVICT_CLEAN_HARD 0x002 /* Clean % blocking app threads */ #define WT_CACHE_EVICT_DIRTY 0x004 /* Evict dirty pages */ #define WT_CACHE_EVICT_DIRTY_HARD 0x008 /* Dirty % blocking app threads */ -#define WT_CACHE_EVICT_SCRUB 0x010 /* Scrub dirty pages pages */ +#define WT_CACHE_EVICT_SCRUB 0x010 /* Scrub dirty pages */ #define WT_CACHE_EVICT_URGENT 0x020 /* Pages are in the urgent queue */ #define WT_CACHE_EVICT_ALL (WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_DIRTY) #define WT_CACHE_EVICT_MASK 0x0FF diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i index b5605769f1a..4255d04ec37 100644 --- a/src/third_party/wiredtiger/src/include/cache.i +++ b/src/third_party/wiredtiger/src/include/cache.i @@ -193,7 +193,7 @@ __wt_cache_bytes_other(WT_CACHE *cache) * __wt_session_can_wait -- * Return if a session available for a potentially slow operation. */ -static inline int +static inline bool __wt_session_can_wait(WT_SESSION_IMPL *session) { /* @@ -202,17 +202,71 @@ __wt_session_can_wait(WT_SESSION_IMPL *session) * the system cache. */ if (!F_ISSET(session, WT_SESSION_CAN_WAIT)) - return (0); + return (false); /* * LSM sets the no-eviction flag when holding the LSM tree lock, in that * case, or when holding the schema lock, we don't want to highjack the * thread for eviction. */ - if (F_ISSET(session, WT_SESSION_NO_EVICTION | WT_SESSION_LOCKED_SCHEMA)) - return (0); + return (!F_ISSET( + session, WT_SESSION_NO_EVICTION | WT_SESSION_LOCKED_SCHEMA)); +} + +/* + * __wt_eviction_clean_needed -- + * Return if an application thread should do eviction due to the total + * volume of dirty data in cache. + */ +static inline bool +__wt_eviction_clean_needed(WT_SESSION_IMPL *session, u_int *pct_fullp) +{ + WT_CACHE *cache; + uint64_t bytes_inuse, bytes_max; + + cache = S2C(session)->cache; + + /* + * Avoid division by zero if the cache size has not yet been set in a + * shared cache. + */ + bytes_max = S2C(session)->cache_size + 1; + bytes_inuse = __wt_cache_bytes_inuse(cache); + + if (pct_fullp != NULL) + *pct_fullp = (u_int)((100 * bytes_inuse) / bytes_max); + + return (bytes_inuse > (cache->eviction_trigger * bytes_max) / 100); +} + +/* + * __wt_eviction_dirty_needed -- + * Return if an application thread should do eviction due to the total + * volume of dirty data in cache. + */ +static inline bool +__wt_eviction_dirty_needed(WT_SESSION_IMPL *session, u_int *pct_fullp) +{ + WT_CACHE *cache; + double dirty_trigger; + uint64_t dirty_inuse, bytes_max; + + cache = S2C(session)->cache; + + /* + * Avoid division by zero if the cache size has not yet been set in a + * shared cache. + */ + bytes_max = S2C(session)->cache_size + 1; + dirty_inuse = __wt_cache_dirty_leaf_inuse(cache); + + if (pct_fullp != NULL) + *pct_fullp = (u_int)((100 * dirty_inuse) / bytes_max); + + if ((dirty_trigger = cache->eviction_scrub_limit) < 1.0) + dirty_trigger = (double)cache->eviction_dirty_trigger; - return (1); + return (dirty_inuse > (uint64_t)(dirty_trigger * bytes_max) / 100); } /* @@ -223,42 +277,30 @@ __wt_session_can_wait(WT_SESSION_IMPL *session) static inline bool __wt_eviction_needed(WT_SESSION_IMPL *session, bool busy, u_int *pct_fullp) { - WT_CONNECTION_IMPL *conn; WT_CACHE *cache; - double dirty_trigger; - uint64_t bytes_inuse, bytes_max, dirty_inuse; u_int pct_dirty, pct_full; + bool clean_needed, dirty_needed; - conn = S2C(session); - cache = conn->cache; + cache = S2C(session)->cache; /* * If the connection is closing we do not need eviction from an * application thread. The eviction subsystem is already closed. */ - if (F_ISSET(conn, WT_CONN_CLOSING)) + if (F_ISSET(S2C(session), WT_CONN_CLOSING)) return (false); - /* - * Avoid division by zero if the cache size has not yet been set in a - * shared cache. - */ - bytes_max = conn->cache_size + 1; - bytes_inuse = __wt_cache_bytes_inuse(cache); - dirty_inuse = __wt_cache_dirty_leaf_inuse(cache); + clean_needed = __wt_eviction_clean_needed(session, &pct_full); + dirty_needed = __wt_eviction_dirty_needed(session, &pct_dirty); /* * Calculate the cache full percentage; anything over the trigger means * we involve the application thread. */ - if (pct_fullp != NULL) { - pct_full = (u_int)((100 * bytes_inuse) / bytes_max); - pct_dirty = (u_int)((100 * dirty_inuse) / bytes_max); - + if (pct_fullp != NULL) *pct_fullp = (u_int)WT_MAX(0, 100 - WT_MIN( (int)cache->eviction_trigger - (int)pct_full, (int)cache->eviction_dirty_trigger - (int)pct_dirty)); - } /* * Only check the dirty trigger when the session is not busy. @@ -268,11 +310,7 @@ __wt_eviction_needed(WT_SESSION_IMPL *session, bool busy, u_int *pct_fullp) * The next transaction in this session will not be able to start until * the cache is under the limit. */ - if ((dirty_trigger = cache->eviction_scrub_limit) < 1.0) - dirty_trigger = (double)cache->eviction_dirty_trigger; - return (bytes_inuse > (cache->eviction_trigger * bytes_max) / 100 || - (!busy && - dirty_inuse > (uint64_t)(dirty_trigger * bytes_max) / 100)); + return (clean_needed || (!busy && dirty_needed)); } /* diff --git a/src/third_party/wiredtiger/src/include/thread_group.h b/src/third_party/wiredtiger/src/include/thread_group.h index f946dcab144..76758a090c4 100644 --- a/src/third_party/wiredtiger/src/include/thread_group.h +++ b/src/third_party/wiredtiger/src/include/thread_group.h @@ -14,7 +14,14 @@ struct __wt_thread { WT_SESSION_IMPL *session; u_int id; wt_thread_t tid; -#define WT_THREAD_RUN 0x01 + + /* + * WT_THREAD and thread-group function flags, merged because + * WT_THREAD_PANIC_FAIL appears in both groups. + */ +#define WT_THREAD_CAN_WAIT 0x01 /* WT_SESSION_CAN_WAIT */ +#define WT_THREAD_PANIC_FAIL 0x02 /* panic if the thread fails */ +#define WT_THREAD_RUN 0x04 /* thread is running */ uint32_t flags; /* The runner function used by all threads. */ @@ -22,12 +29,6 @@ struct __wt_thread { }; /* - * Flags for thread group functions. - */ -#define WT_THREAD_CAN_WAIT 0x01 -#define WT_THREAD_PANIC_FAIL 0x02 - -/* * WT_THREAD_GROUP -- * Encapsulation of a group of utility threads. */ diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index 8ac8f5f9f6d..46b63ac6129 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -166,9 +166,10 @@ __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) /* * If this was the last hazard pointer in the session, - * we may need to update our transactional context. + * reset the size so that checks can skip this session. */ - --session->nhazard; + if (--session->nhazard == 0) + WT_PUBLISH(session->hazard_size, 0); return (0); } diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 180a06a3aed..3aad95f5a9f 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -423,7 +423,8 @@ __checkpoint_reduce_dirty_cache(WT_SESSION_IMPL *session) * level. */ __wt_sleep(0, 10 * stepdown_us); - cache->eviction_scrub_limit = current_dirty - delta; + cache->eviction_scrub_limit = + WT_MAX(cache->eviction_dirty_target, current_dirty - delta); WT_STAT_CONN_SET(session, txn_checkpoint_scrub_target, cache->eviction_scrub_limit); WT_RET(__wt_epoch(session, &last)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c index 1ff7b10e1c6..b67dae6d647 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c @@ -27,6 +27,8 @@ */ #include "test_util.h" +#include <signal.h> + /* * JIRA ticket reference: WT-2719 * Test case description: Fuzz testing for WiredTiger reconfiguration. @@ -191,6 +193,41 @@ handle_message(WT_EVENT_HANDLER *handler, static WT_EVENT_HANDLER event_handler = { NULL, handle_message, NULL, NULL }; +static const char *current; /* Current test configuration */ + +static void on_alarm(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void +on_alarm(int signo) +{ + (void)signo; /* Unused parameter */ + + fprintf(stderr, "configuration timed out: %s\n", current); + abort(); + + /* NOTREACHED */ +} + +static void +reconfig(TEST_OPTS *opts, WT_SESSION *session, const char *config) +{ + int ret; + + current = config; + + /* + * Reconfiguration starts and stops servers, so hangs are more likely + * here than in other tests. Don't let the test run too long and get + * a core dump when it happens. + */ + (void)alarm(60); + if ((ret = opts->conn->reconfigure(opts->conn, config)) != 0) { + fprintf(stderr, "%s: %s\n", + config, session->strerror(session, ret)); + exit (EXIT_FAILURE); + } + (void)alarm(0); +} + int main(int argc, char *argv[]) { @@ -200,7 +237,6 @@ main(int argc, char *argv[]) WT_SESSION *session; size_t len; u_int i, j; - int ret; const char *p; char *config; @@ -226,13 +262,12 @@ main(int argc, char *argv[]) len = WT_ELEMENTS(list) * 64; config = dmalloc(len); + /* Set an alarm so we can debug hangs. */ + (void)signal(SIGALRM, on_alarm); + /* A linear pass through the list. */ for (i = 0; i < WT_ELEMENTS(list); ++i) - if ((ret = opts->conn->reconfigure(opts->conn, list[i])) != 0) { - fprintf(stderr, "%s: %s\n", - list[i], session->strerror(session, ret)); - return (EXIT_FAILURE); - } + reconfig(opts, session, list[i]); /* * A linear pass through the list, adding random elements. @@ -264,11 +299,7 @@ main(int argc, char *argv[]) } strcat(config, p); } - if ((ret = opts->conn->reconfigure(opts->conn, config)) != 0) { - fprintf(stderr, "%s: %s\n", - config, session->strerror(session, ret)); - return (EXIT_FAILURE); - } + reconfig(opts, session, config); } /* diff --git a/src/third_party/wiredtiger/test/suite/test_compact01.py b/src/third_party/wiredtiger/test/suite/test_compact01.py index 861e957d18d..8da1a0df4da 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact01.py +++ b/src/third_party/wiredtiger/test/suite/test_compact01.py @@ -54,9 +54,11 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): ('utility', dict(utility=1,reopen=0)), ] scenarios = make_scenarios(types, compact) - # We want a large cache so that eviction doesn't happen - # (which could skew our compaction results). - conn_config = 'cache_size=250MB,statistics=(all)' + + # Configure the connection so that eviction doesn't happen (which could + # skew our compaction results). + conn_config = 'cache_size=1GB,eviction_checkpoint_target=80,' +\ + 'eviction_dirty_target=80,eviction_dirty_trigger=95,statistics=(all)' # Test compaction. def test_compact(self): |