diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2019-07-12 16:51:32 +1000 |
|---|---|---|
| committer | Luke Chen <luke.chen@mongodb.com> | 2019-07-12 16:51:32 +1000 |
| commit | 67b760c562d7f189bad589841b4dcd14acf702d9 (patch) | |
| tree | bd2291d5306357a1093125f19ae01d5c03d9547c | |
| parent | 9ae337bd27f7a513df548256400596a6eba4d7a3 (diff) | |
| download | mongo-67b760c562d7f189bad589841b4dcd14acf702d9.tar.gz | |
Import wiredtiger: 3f686382114354b29b1d92b9c4a7dfc870dc5b94 from branch mongodb-4.2
ref: 99e0760cc5..3f68638211
for: 4.3.1
WT-4530 WiredTiger session statistics cursor returns incorrect key on BigEndian systems
WT-4612 Improve test coverage for prepare updates older than the oldest
WT-4791 Coverity: Dereferencing null (cbt->ins)
WT-4842 Enhance lookaside score calculation
WT-4857 Fix a bug in column store where skip list traversal could miss an entry
WT-4889 Dump additional transaction fields
WT-4913 Fix the Windows CRC32 on blocks that aren't 8B aligned and/or multiples of 8B
WT-4916 Clang Format pre-formatting fixes
WT-4926 Fix the WiredTiger static test suite's test for EBUSY failures
WT-4927 Allow updates with timestamps to be evicted to lookaside
WT-4928 Fix clang scan warnings in Jenkins Pull Request testing
83 files changed, 938 insertions, 314 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 45f81ee125d..f895a53c426 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -526,6 +526,14 @@ t = env.Program("wtperf", [ LIBS=[wtlib, shim, testutil] + wtlibs) Default(t) +t = env.Program('wt2695_checksum', ['test/csuite/wt2695_checksum/main.c'], + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) + +t = env.Program('wt4117_checksum', ['test/csuite/wt4117_checksum/main.c'], + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) + #Build the Examples for ex in examples: exp = env.Program(ex, "examples/c/" + ex + ".c", LIBS=[wtlib, shim, testutil] + wtlibs) diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index c529ebc0366..1d398a4aa88 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -55,6 +55,7 @@ src/checksum/power8/crc32.sx POWERPC_HOST src/checksum/power8/crc32_wrapper.c POWERPC_HOST src/checksum/software/checksum.c src/checksum/x86/crc32-x86.c X86_HOST +src/checksum/x86/crc32-x86-alt.c X86_HOST src/checksum/zseries/crc32-s390x.c ZSERIES_HOST src/checksum/zseries/crc32le-vx.sx ZSERIES_HOST src/config/config.c diff --git a/src/third_party/wiredtiger/dist/log.py b/src/third_party/wiredtiger/dist/log.py index b3e6a71b63a..c076a023f11 100644 --- a/src/third_party/wiredtiger/dist/log.py +++ b/src/third_party/wiredtiger/dist/log.py @@ -333,7 +333,7 @@ for optype in log_data.optypes: }) tfile.write(''' -\tWT_ILLEGAL_VALUE(session, optype); +\tdefault:\n\t\treturn (__wt_illegal_value(session, optype)); \t} \treturn (0); diff --git a/src/third_party/wiredtiger/dist/s_clang-scan b/src/third_party/wiredtiger/dist/s_clang-scan index 33d6f3e7d9e..72616f9e9d4 100644 --- a/src/third_party/wiredtiger/dist/s_clang-scan +++ b/src/third_party/wiredtiger/dist/s_clang-scan @@ -1,50 +1,52 @@ #! /bin/sh t=__wt.$$ -trap 'rm -rf $t /tmp/scan-build-*' 0 1 2 3 13 15 +trap 'rm -rf $t' 0 1 2 3 13 15 -# Installation of the clang development package isn't standard, list a -# couple of the places we're using. -export PATH=$PATH:/usr/local/clang50/bin:/usr/local/llvm-devel/bin - -# Remove old reports. -rm -rf /tmp/scan-build-* +set -e # Find the top-level WiredTiger directory and move to there. -p="$PWD" -while test "$p" != "/" ; do - if test -d "$p/build_posix"; then - break; - fi - p=`dirname $p` -done -test "$p" != "/" || { - echo "$0: cannot find the WiredTiger top-level directory" +top=`git rev-parse --show-toplevel` && cd $top || exit 1 +echo "$0: running scan-build in $PWD" + +# Clang isn't installed in any standard place, find a binary we can use. +# If the caller gives us $SCANBUILD, check for $SCANCOMPILER and default +# to "clang". +scan="$SCANBUILD" +test -z "$scan" || compiler="${SCANCOMPILER:-clang}" + +if test -z "$scan" && + test -x /usr/bin/clang && + test -x /usr/lib/llvm-3.9/bin/scan-build; then + compiler="/usr/bin/clang" + scan="/usr/lib/llvm-3.9/bin/scan-build" +fi + +if test -z "$scan"; then + echo "$0: no clang compiler and scan-build programs found" + echo "$0: set \$SCANCOMPILER=/path/clang and \$SCANBUILD=/path/scan-build to continue" exit 1 -} +fi +echo "$0: compiler: $compiler" +echo "$0: scan-build: $scan" -cd $p || exit 1 -echo "$0: running scan-build in $p..." +# Remove old reports. +rm -rf clangScanBuildReports && mkdir clangScanBuildReports sh autogen.sh > /dev/null || exit 1 -args="--use-cc=clang40 -no-failure-reports" +args="-o clangScanBuildReports" +args="$args --use-cc=$compiler" args="$args -disable-checker core.NullDereference" -scan-build $args ./configure 'CFLAGS=-g -Wall' \ +$scan $args ./configure 'CFLAGS=-g' \ --disable-shared --enable-diagnostic --enable-strict > /dev/null -scan-build $args make -j 8 wt 2>&1 > $t +$scan $args make -j 8 wt 1>/dev/null 2>$t make -j 8 distclean > /dev/null -grep 'No bugs found' $t > /dev/null || { - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo 'unexpected scan-build output' - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - sed -e '/^--- /d' \ - -e '/^\/bin\/sh /d' \ - -e '/^depbase=/d' \ - -e '/^libtool: /d' < $t - exit 1 -} - -exit 0 +grep 'No bugs found' $t > /dev/null && exit 0 +echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" +echo 'scan-build output:' +echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" +cat $t +exit 1 diff --git a/src/third_party/wiredtiger/dist/s_function b/src/third_party/wiredtiger/dist/s_function index 314a8c5bb06..396e8dee02e 100755 --- a/src/third_party/wiredtiger/dist/s_function +++ b/src/third_party/wiredtiger/dist/s_function @@ -23,7 +23,7 @@ file_parse() # where there's a jump to the error label after the error label. for f in `find bench examples ext src test -name '*.[ci]'`; do file_parse $f | - egrep '(WT_ERR[_A-Z]*|WT_ILLEGAL_VALUE_ERR)\(.*(WT_ILLEGAL_VALUE|WT_RET[_A-Z]*)\(.*err:|[^a-z_]err:.*(WT_ERR|WT_ILLEGAL_VALUE_ERR)\(' | + egrep '(WT_ERR[_A-Z]*)\(.*(WT_RET[_A-Z]*)\(.*err:|[^a-z_]err:.*(WT_ERR)\(' | sed 's/:.*//' > $t test -s $t && { @@ -37,7 +37,7 @@ done for f in `find bench examples ext src test -name '*.[ci]'`; do file_parse $f | sed "s=^=$f:=" done | python dist/s_function_loop.py | - egrep '\{@[^@]*(WT_ILLEGAL_VALUE|WT_RET[_A-Z]*)\([^@]*(WT_ERR[_A-Z]*|WT_ILLEGAL_VALUE_ERR)\(.*err:' | + egrep '\{@[^@]*(WT_RET[_A-Z]*)\([^@]*(WT_ERR[_A-Z]*)\(.*err:' | sed -e 's/^\([^:]*\): *\([^:]*\):.*/\1:\2: mix of returns and jump to the error label within a loop/' # Return of 0 in functions after a jump to the error label. diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index 7313201c678..bda8bba4d6a 100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -147,7 +147,6 @@ for f in `find bench ext src test -name '*.[ci]'`; do -e '/WT_CURSOR_NEEDKEY(/d' \ -e '/WT_CURSOR_NEEDVALUE(/d' \ -e '/WT_ERR[A-Z_]*(/d' \ - -e '/WT_ILLEGAL_VALUE[A-Z_]*(/d' \ -e '/WT_PANIC[A-Z_]*(/d' \ -e '/WT_RET[A-Z_]*(/d' \ -e '/WT_SIZE_CHECK_PACK(/d' \ @@ -176,7 +175,6 @@ for f in `find bench ext src test -name '*.[ci]'`; do -e '/WT_CURSOR_NEEDKEY/d' \ -e '/WT_CURSOR_NEEDVALUE/d' \ -e '/WT_ERR/d' \ - -e '/WT_ILLEGAL_VALUE_ERR/d' \ -e '/WT_SYSCALL.*ret/d' \ -e '/WT_TRET/d' \ -e 's/^\([^(]*\).*/\1/' \ diff --git a/src/third_party/wiredtiger/examples/c/ex_stat.c b/src/third_party/wiredtiger/examples/c/ex_stat.c index c389dc5c82e..3b4389c90a0 100644 --- a/src/third_party/wiredtiger/examples/c/ex_stat.c +++ b/src/third_party/wiredtiger/examples/c/ex_stat.c @@ -30,7 +30,7 @@ */ #include <test_util.h> -void get_stat(WT_CURSOR *cursor, int stat_field, uint64_t *valuep); +void get_stat(WT_CURSOR *cursor, int stat_field, int64_t *valuep); void print_cursor(WT_CURSOR *); void print_database_stats(WT_SESSION *); void print_derived_stats(WT_SESSION *); @@ -46,7 +46,7 @@ void print_cursor(WT_CURSOR *cursor) { const char *desc, *pvalue; - uint64_t value; + int64_t value; int ret; while ((ret = cursor->next(cursor)) == 0) { @@ -134,7 +134,7 @@ print_overflow_pages(WT_SESSION *session) /*! [statistics retrieve by key] */ WT_CURSOR *cursor; const char *desc, *pvalue; - uint64_t value; + int64_t value; error_check(session->open_cursor(session, "statistics:table:access", NULL, NULL, &cursor)); @@ -150,7 +150,7 @@ print_overflow_pages(WT_SESSION *session) /*! [statistics calculation helper function] */ void -get_stat(WT_CURSOR *cursor, int stat_field, uint64_t *valuep) +get_stat(WT_CURSOR *cursor, int stat_field, int64_t *valuep) { const char *desc, *pvalue; @@ -172,20 +172,20 @@ print_derived_stats(WT_SESSION *session) { /*! [statistics calculate table fragmentation] */ - uint64_t ckpt_size, file_size, percent; + int64_t ckpt_size, file_size, percent; get_stat(cursor, WT_STAT_DSRC_BLOCK_CHECKPOINT_SIZE, &ckpt_size); get_stat(cursor, WT_STAT_DSRC_BLOCK_SIZE, &file_size); percent = 0; if (file_size != 0) percent = 100 * ((file_size - ckpt_size) / file_size); - printf("Table is %" PRIu64 "%% fragmented\n", percent); + printf("Table is %" PRId64 "%% fragmented\n", percent); /*! [statistics calculate table fragmentation] */ } { /*! [statistics calculate write amplification] */ - uint64_t app_insert, app_remove, app_update, fs_writes; + int64_t app_insert, app_remove, app_update, fs_writes; get_stat(cursor, WT_STAT_DSRC_CURSOR_INSERT_BYTES, &app_insert); get_stat(cursor, WT_STAT_DSRC_CURSOR_REMOVE_BYTES, &app_remove); diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 5e4ee3db6a4..051c9f97bca 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,6 +1,6 @@ { - "vendor": "wiredtiger", - "github": "wiredtiger/wiredtiger.git", - "branch": "mongodb-4.2", - "commit": "99e0760cc5f11440b21184874d8dd2ae5bde23d6" + "commit": "3f686382114354b29b1d92b9c4a7dfc870dc5b94", + "github": "wiredtiger/wiredtiger.git", + "vendor": "wiredtiger", + "branch": "mongodb-4.2" } diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index a968370c2a5..9b7a42b5b9c 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -129,7 +129,8 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ERR(__wt_block_truncate(session, block, ci->file_size)); if (0) { -err: /* +err: + /* * Don't call checkpoint-unload: unload does real work including * file truncation. If we fail early enough that the checkpoint * information isn't correct, bad things would happen. The only @@ -357,11 +358,6 @@ __ckpt_verify(WT_SESSION_IMPL *session, WT_CKPT *ckptbase) break; /* FALLTHROUGH */ default: - /* - * Don't convert to WT_ILLEGAL_VALUE, it won't compile - * on some gcc compilers because they don't understand - * FALLTHROUGH as part of a macro. - */ return (__wt_illegal_value(session, ckpt->flags)); } return (0); diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 9e4689d7b7e..554a485ce2f 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -319,7 +319,8 @@ __desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block) WT_BLOCK_DESC *desc; WT_DECL_ITEM(buf); WT_DECL_RET; - uint32_t checksum_calculate, checksum_tmp; + uint32_t checksum_saved, checksum_tmp; + bool checksum_matched; /* If in-memory, we don't read or write the descriptor structure. */ if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) @@ -340,10 +341,13 @@ __desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block) * a calculated checksum that should match the checksum in the header. */ desc = buf->mem; - checksum_tmp = desc->checksum; + checksum_saved = checksum_tmp = desc->checksum; +#ifdef WORDS_BIGENDIAN + checksum_tmp = __wt_bswap32(checksum_tmp); +#endif desc->checksum = 0; - checksum_calculate = __wt_checksum(desc, allocsize); - desc->checksum = checksum_tmp; + checksum_matched = __wt_checksum_match(desc, allocsize, checksum_tmp); + desc->checksum = checksum_saved; __wt_block_desc_byteswap(desc); /* @@ -355,8 +359,7 @@ __desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block) * may have entered the wrong file name, and is now frantically pounding * their interrupt key. */ - if (desc->magic != WT_BLOCK_MAGIC || - desc->checksum != checksum_calculate) + if (desc->magic != WT_BLOCK_MAGIC || !checksum_matched) WT_ERR_MSG(session, WT_ERROR, "%s does not appear to be a WiredTiger file", block->name); diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 0c2ef31254f..ce2aa031b8f 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -224,7 +224,6 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, { WT_BLOCK_HEADER *blk, swap; size_t bufsize; - uint32_t page_checksum; __wt_verbose(session, WT_VERB_READ, "off %" PRIuMAX ", size %" PRIu32 ", checksum %#" PRIx32, @@ -248,9 +247,6 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, F_SET(buf, WT_ITEM_ALIGNED); bufsize = WT_MAX(size, buf->memsize + 10); } - WT_RET(__wt_buf_init(session, buf, bufsize)); - WT_RET(__wt_read(session, block->fh, offset, size, buf->mem)); - buf->size = size; /* * Ensure we don't read information that isn't there. It shouldn't ever @@ -262,6 +258,10 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, "allocation size of %" PRIu32, block->name, size, block->allocsize); + WT_RET(__wt_buf_init(session, buf, bufsize)); + WT_RET(__wt_read(session, block->fh, offset, size, buf->mem)); + buf->size = size; + /* * We incrementally read through the structure before doing a checksum, * do little- to big-endian handling early on, and then select from the @@ -271,10 +271,9 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, __wt_block_header_byteswap_copy(blk, &swap); if (swap.checksum == checksum) { blk->checksum = 0; - page_checksum = __wt_checksum(buf->mem, + if (__wt_checksum_match(buf->mem, F_ISSET(&swap, WT_BLOCK_DATA_CKSUM) ? - size : WT_BLOCK_COMPRESS_SKIP); - if (page_checksum == checksum) { + size : WT_BLOCK_COMPRESS_SKIP, checksum)) { /* * Swap the page-header as needed; this doesn't belong * here, but it's the best place to catch all callers. @@ -287,10 +286,8 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, __wt_errx(session, "%s: read checksum error for %" PRIu32 "B block at " "offset %" PRIuMAX ": calculated block checksum " - "of %#" PRIx32 " doesn't match expected checksum " - "of %#" PRIx32, - block->name, - size, (uintmax_t)offset, page_checksum, checksum); + " doesn't match expected checksum", + block->name, size, (uintmax_t)offset); } else if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) __wt_errx(session, diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 94fe29d468d..7744ebc40eb 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -214,7 +214,8 @@ __cursor_var_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) __cursor_set_recno(cbt, cbt->recno + 1); new_page: -restart_read: /* Find the matching WT_COL slot. */ +restart_read: + /* Find the matching WT_COL slot. */ if ((cip = __col_var_search(cbt->ref, cbt->recno, &rle_start)) == NULL) return (WT_NOTFOUND); @@ -497,7 +498,8 @@ __wt_cursor_key_order_check( return (__cursor_key_order_check_col(session, cbt, next)); case WT_PAGE_ROW_LEAF: return (__cursor_key_order_check_row(session, cbt, next)); - WT_ILLEGAL_VALUE(session, cbt->ref->page->type); + default: + return (__wt_illegal_value(session, cbt->ref->page->type)); } /* NOTREACHED */ } @@ -522,7 +524,8 @@ __wt_cursor_key_order_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) case WT_PAGE_ROW_LEAF: return (__wt_buf_set(session, cbt->lastkey, cbt->iface.key.data, cbt->iface.key.size)); - WT_ILLEGAL_VALUE(session, cbt->ref->page->type); + default: + return (__wt_illegal_value(session, cbt->ref->page->type)); } /* NOTREACHED */ } @@ -666,7 +669,8 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) ret = __cursor_var_append_next( cbt, newpage, restart); break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } if (ret == 0 || ret == WT_PREPARE_CONFLICT) break; @@ -684,7 +688,8 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) case WT_PAGE_ROW_LEAF: ret = __cursor_row_next(cbt, newpage, restart); break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } if (ret != WT_NOTFOUND) break; diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 5ac53cb2036..45e0ac153a3 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -38,6 +38,7 @@ __cursor_skip_prev(WT_CURSOR_BTREE *cbt) WT_INSERT *current, *ins; WT_ITEM key; WT_SESSION_IMPL *session; + uint64_t recno; int i; session = (WT_SESSION_IMPL *)cbt->iface.session; @@ -47,6 +48,7 @@ restart: * If the search stack does not point at the current item, fill it in * with a search. */ + recno = WT_INSERT_RECNO(cbt->ins); while ((current = cbt->ins) != PREV_INS(cbt, 0)) { if (cbt->btree->type == BTREE_ROW) { key.data = WT_INSERT_KEY(current); @@ -55,8 +57,7 @@ restart: session, cbt, cbt->ins_head, &key)); } else cbt->ins = __col_insert_search(cbt->ins_head, - cbt->ins_stack, cbt->next_stack, - WT_INSERT_RECNO(current)); + cbt->ins_stack, cbt->next_stack, recno); } /* @@ -360,7 +361,8 @@ __cursor_var_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) new_page: if (cbt->recno < cbt->ref->ref_recno) return (WT_NOTFOUND); -restart_read: /* Find the matching WT_COL slot. */ +restart_read: + /* Find the matching WT_COL slot. */ if ((cip = __col_var_search(cbt->ref, cbt->recno, &rle_start)) == NULL) return (WT_NOTFOUND); @@ -629,7 +631,8 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) ret = __cursor_var_append_prev( cbt, newpage, restart); break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } if (ret == 0 || ret == WT_PREPARE_CONFLICT) break; @@ -649,7 +652,8 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) case WT_PAGE_ROW_LEAF: ret = __cursor_row_prev(cbt, newpage, restart); break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } if (ret != WT_NOTFOUND) break; diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 51ccf1d6415..4d94bcdb23e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -1093,7 +1093,8 @@ __wt_btcur_remove(WT_CURSOR_BTREE *cbt, bool positioned) goto err; } -retry: /* +retry: + /* * Note these steps must be repeatable, we'll continue to take this path * as long as we encounter WT_RESTART. * @@ -1215,7 +1216,8 @@ search_notfound: ret = WT_NOTFOUND; __cursor_state_restore(cursor, &state); } -done: /* +done: + /* * Upper level cursor removes don't expect the cursor value to be set * after a successful remove (and check in diagnostic mode). Error * handling may have converted failure to a success, do a final check. @@ -1376,7 +1378,8 @@ done: switch (modify_type) { break; case WT_UPDATE_BIRTHMARK: case WT_UPDATE_TOMBSTONE: - WT_ILLEGAL_VALUE(session, modify_type); + default: + return (__wt_illegal_value(session, modify_type)); } } diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index ef40849280a..685fb983718 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -458,7 +458,8 @@ __wt_debug_disk( case WT_PAGE_OVFL: WT_ERR(ds->f(ds, ", datalen %" PRIu32, dsk->u.datalen)); break; - WT_ILLEGAL_VALUE_ERR(session, dsk->type); + default: + WT_ERR(__wt_illegal_value(session, dsk->type)); } if (F_ISSET(dsk, WT_PAGE_COMPRESSED)) @@ -790,7 +791,8 @@ __debug_page(WT_DBG *ds, WT_REF *ref, uint32_t flags) if (LF_ISSET(WT_DEBUG_TREE_LEAF)) WT_RET(__debug_page_row_leaf(ds, ref->page)); break; - WT_ILLEGAL_VALUE(session, ref->page->type); + default: + return (__wt_illegal_value(session, ref->page->type)); } return (0); @@ -840,7 +842,8 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) case WT_PAGE_ROW_LEAF: entries = page->entries; break; - WT_ILLEGAL_VALUE(session, page->type); + default: + return (__wt_illegal_value(session, page->type)); } WT_RET(ds->f(ds, ": %s\n", __wt_page_type_string(page->type))); @@ -880,7 +883,8 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) break; case 0: break; - WT_ILLEGAL_VALUE(session, mod->rec_result); + default: + return (__wt_illegal_value(session, mod->rec_result)); } if (split_gen != 0) WT_RET(ds->f(ds, ", split-gen=%" PRIu64, split_gen)); @@ -965,13 +969,14 @@ __debug_page_col_int(WT_DBG *ds, WT_PAGE *page, uint32_t flags) WT_RET(__debug_ref(ds, ref)); } WT_INTL_FOREACH_END; - if (LF_ISSET(WT_DEBUG_TREE_WALK)) + if (LF_ISSET(WT_DEBUG_TREE_WALK)) { WT_INTL_FOREACH_BEGIN(session, page, ref) { if (ref->state == WT_REF_MEM) { WT_RET(ds->f(ds, "\n")); WT_RET(__debug_page(ds, ref, flags)); } } WT_INTL_FOREACH_END; + } return (0); } @@ -1039,13 +1044,14 @@ __debug_page_row_int(WT_DBG *ds, WT_PAGE *page, uint32_t flags) WT_RET(__debug_ref(ds, ref)); } WT_INTL_FOREACH_END; - if (LF_ISSET(WT_DEBUG_TREE_WALK)) + if (LF_ISSET(WT_DEBUG_TREE_WALK)) { WT_INTL_FOREACH_BEGIN(session, page, ref) { if (ref->state == WT_REF_MEM) { WT_RET(ds->f(ds, "\n")); WT_RET(__debug_page(ds, ref, flags)); } } WT_INTL_FOREACH_END; + } return (0); } @@ -1442,7 +1448,8 @@ __debug_cell_data(WT_DBG *ds, case WT_CELL_VALUE_SHORT: WT_ERR(__debug_item_value(ds, tag, buf->data, buf->size)); break; - WT_ILLEGAL_VALUE_ERR(session, unpack->raw); + default: + WT_ERR(__wt_illegal_value(session, unpack->raw)); } err: __wt_scr_free(session, &buf); diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index 00d6bbc68e8..13c2a2be67c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -209,7 +209,8 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) case WT_REF_LIMBO: case WT_REF_LOOKASIDE: case WT_REF_READING: - WT_ILLEGAL_VALUE(session, current_state); + default: + return (__wt_illegal_value(session, current_state)); } if (locked) @@ -243,7 +244,8 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) WT_REF_SET_STATE(ref, current_state); -done: /* +done: + /* * Now mark the truncate aborted: this must come last because after * this point there is nothing preventing the page from being evicted. */ @@ -466,7 +468,8 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) return (0); -err: /* +err: + /* * The page-delete update structure may have existed before we were * called, and presumably might be in use by a running transaction. * The list of update structures cannot have been created before we diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index 10b44d85c67..50184189b86 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -120,7 +120,7 @@ __wt_bt_read(WT_SESSION_IMPL *session, fail_msg = "block decompression failed"; goto corrupt; } - } else + } else { /* * If we uncompressed above, the page is in the correct buffer. * If we get here the data may be in the wrong buffer and the @@ -130,6 +130,7 @@ __wt_bt_read(WT_SESSION_IMPL *session, if (ip != NULL) WT_ERR(__wt_buf_set( session, buf, ip->data, dsk->mem_size)); + } /* If the handle is a verify handle, verify the physical page. */ if (F_ISSET(btree, WT_BTREE_VERIFY)) { diff --git a/src/third_party/wiredtiger/src/btree/bt_ovfl.c b/src/third_party/wiredtiger/src/btree/bt_ovfl.c index e254a9acf7d..fb64fc1f7a6 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ovfl.c +++ b/src/third_party/wiredtiger/src/btree/bt_ovfl.c @@ -241,7 +241,8 @@ __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell) __wt_cell_type_reset(session, unpack->cell, WT_CELL_VALUE_OVFL, WT_CELL_VALUE_OVFL_RM); break; - WT_ILLEGAL_VALUE(session, unpack->raw); + default: + return (__wt_illegal_value(session, unpack->raw)); } __wt_writeunlock(session, &btree->ovfl_lock); diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index 42ab5056ad8..de7c6243a3b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -58,7 +58,8 @@ __wt_page_alloc(WT_SESSION_IMPL *session, */ size += alloc_entries * sizeof(WT_ROW); break; - WT_ILLEGAL_VALUE(session, type); + default: + return (__wt_illegal_value(session, type)); } WT_RET(__wt_calloc(session, 1, size, &page)); @@ -113,7 +114,8 @@ err: if ((pindex = WT_INTL_INDEX_GET_SAFE(page)) != NULL) { NULL : (WT_ROW *)((uint8_t *)page + sizeof(WT_PAGE)); page->entries = alloc_entries; break; - WT_ILLEGAL_VALUE(session, type); + default: + return (__wt_illegal_value(session, type)); } /* Increment the cache statistics. */ @@ -190,7 +192,8 @@ __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, WT_RET(__inmem_row_leaf_entries( session, dsk, &alloc_entries)); break; - WT_ILLEGAL_VALUE(session, dsk->type); + default: + return (__wt_illegal_value(session, dsk->type)); } /* Allocate and initialize a new WT_PAGE. */ @@ -227,7 +230,8 @@ __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, case WT_PAGE_ROW_LEAF: WT_ERR(__inmem_row_leaf(session, page, check_unstable)); break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } /* Update the page's cache statistics. */ @@ -511,7 +515,8 @@ __inmem_row_int(WT_SESSION_IMPL *session, WT_PAGE *page, size_t *sizep) ref->addr = unpack.cell; ++refp; break; - WT_ILLEGAL_VALUE_ERR(session, unpack.type); + default: + WT_ERR(__wt_illegal_value(session, unpack.type)); } } WT_CELL_FOREACH_END; @@ -561,7 +566,8 @@ __inmem_row_leaf_entries( case WT_CELL_VALUE: case WT_CELL_VALUE_OVFL: break; - WT_ILLEGAL_VALUE(session, unpack.type); + default: + return (__wt_illegal_value(session, unpack.type)); } } WT_CELL_FOREACH_END; @@ -627,7 +633,8 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page, bool check_unstable) --page->entries; } break; - WT_ILLEGAL_VALUE(session, unpack.type); + default: + return (__wt_illegal_value(session, unpack.type)); } } WT_CELL_FOREACH_END; diff --git a/src/third_party/wiredtiger/src/btree/bt_random.c b/src/third_party/wiredtiger/src/btree/bt_random.c index 6f51bb26e65..255c08e0b60 100644 --- a/src/third_party/wiredtiger/src/btree/bt_random.c +++ b/src/third_party/wiredtiger/src/btree/bt_random.c @@ -192,7 +192,8 @@ __wt_random_descent(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) eviction = LF_ISSET(WT_READ_CACHE); if (0) { -restart: /* +restart: + /* * Discard the currently held page and restart the search from * the root. */ diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 3c15053fc1c..ed94a7298b7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -222,7 +222,8 @@ __las_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) WT_ERR(__wt_buf_set(session, current_key, las_key.data, las_key.size)); break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } /* Append the latest update to the list. */ @@ -252,7 +253,8 @@ __las_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) current_key, ref, &cbt, first_upd)); first_upd = NULL; break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } /* Discard the cursor. */ @@ -579,7 +581,8 @@ skip_read: WT_REF_SET_STATE(ref, final_state); return (ret); -err: /* +err: + /* * If the function building an in-memory version of the page failed, * it discarded the page, but not the disk image. Discard the page * and separately discard the disk image in all cases. @@ -611,7 +614,7 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags uint64_t sleep_usecs, yield_cnt; uint32_t current_state; int force_attempts; - bool busy, cache_work, did_read, stalled, wont_need; + bool busy, cache_work, evict_skip, stalled, wont_need; btree = S2BT(session); @@ -634,7 +637,7 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags WT_STAT_DATA_INCR(session, cache_pages_requested); } - for (did_read = wont_need = stalled = false, + for (evict_skip = stalled = wont_need = false, force_attempts = 0, sleep_usecs = yield_cnt = 0;;) { switch (current_state = ref->state) { case WT_REF_DELETED: @@ -664,7 +667,8 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags if (LF_ISSET(WT_READ_CACHE)) return (WT_NOTFOUND); -read: /* +read: + /* * The page isn't in memory, read it. If this thread * respects the cache size, check for space in the * cache. @@ -680,7 +684,7 @@ read: /* * We just read a page, don't evict it before we have a * chance to use it. */ - did_read = true; + evict_skip = true; /* * If configured to not trash the cache, leave the page @@ -760,7 +764,7 @@ read: /* /* * Check if the page requires forced eviction. */ - if (did_read || LF_ISSET(WT_READ_NO_SPLIT) || + if (evict_skip || LF_ISSET(WT_READ_NO_SPLIT) || btree->evict_disabled > 0 || btree->lsm_primary) goto skip_evict; @@ -780,8 +784,13 @@ read: /* __evict_force_check(session, ref)) { ++force_attempts; ret = __wt_page_release_evict(session, ref, 0); - /* If forced eviction fails, stall. */ - if (ret == EBUSY) { + /* + * If forced eviction succeeded, don't retry. + * If it failed, stall. + */ + if (ret == 0) + evict_skip = true; + else if (ret == EBUSY) { WT_NOT_READ(ret, 0); WT_STAT_CONN_INCR(session, page_forcible_evict_blocked); @@ -800,7 +809,8 @@ read: /* continue; } -skip_evict: /* +skip_evict: + /* * If we read the page and are configured to not trash * the cache, and no other thread has already used the * page, set the read generation so the page is evicted @@ -837,7 +847,8 @@ skip_evict: /* return (LF_ISSET(WT_READ_IGNORE_CACHE_SIZE) && !F_ISSET(session, WT_SESSION_IGNORE_CACHE_SIZE) ? 0 : __wt_txn_autocommit_check(session)); - WT_ILLEGAL_VALUE(session, current_state); + default: + return (__wt_illegal_value(session, current_state)); } /* @@ -856,14 +867,13 @@ skip_evict: /* /* * If stalling and this thread is allowed to do eviction work, - * check if the cache needs help. If we do work for the cache, - * substitute that for a sleep. + * check if the cache needs help evicting clean pages (don't + * force a read to do dirty eviction). If we do work for the + * cache, substitute that for a sleep. */ if (!LF_ISSET(WT_READ_IGNORE_CACHE_SIZE)) { WT_RET(__wt_cache_eviction_check( - session, true, - !F_ISSET(&session->txn, WT_TXN_HAS_ID), - &cache_work)); + session, true, true, &cache_work)); if (cache_work) continue; } diff --git a/src/third_party/wiredtiger/src/btree/bt_rebalance.c b/src/third_party/wiredtiger/src/btree/bt_rebalance.c index 578e40e8864..9c2a42aa4c0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_rebalance.c +++ b/src/third_party/wiredtiger/src/btree/bt_rebalance.c @@ -237,7 +237,8 @@ __rebalance_col_walk(WT_SESSION_IMPL *session, wt_timestamp_t durable_ts, WT_ERR(__rebalance_leaf_append( session, durable_ts, NULL, 0, &unpack, rs)); break; - WT_ILLEGAL_VALUE_ERR(session, unpack.type); + default: + WT_ERR(__wt_illegal_value(session, unpack.type)); } } WT_CELL_FOREACH_END; @@ -384,7 +385,8 @@ __rebalance_row_walk(WT_SESSION_IMPL *session, wt_timestamp_t durable_ts, first_cell = false; break; - WT_ILLEGAL_VALUE_ERR(session, unpack.type); + default: + WT_ERR(__wt_illegal_value(session, unpack.type)); } } WT_CELL_FOREACH_END; @@ -445,7 +447,8 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__rebalance_col_walk( session, WT_TS_MAX, ref->page->dsk, rs)); break; - WT_ILLEGAL_VALUE_ERR(session, rs->type); + default: + WT_ERR(__wt_illegal_value(session, rs->type)); } /* Build a new root page. */ @@ -465,7 +468,8 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) ref->page = rs->root; rs->root = NULL; -err: /* Discard any leftover root page we created. */ +err: + /* Discard any leftover root page we created. */ if (rs->root != NULL) { __wt_page_modify_clear(session, rs->root); __wt_page_out(session, &rs->root); diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 830a9e19f9f..98ca6bf91e4 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -1037,7 +1037,8 @@ __slvg_col_range_overlap( * discard b_trk. */ if (a_trk->trk_gen > b_trk->trk_gen) { -delete_b: /* +delete_b: + /* * After page and overflow reconciliation, one (and only one) * page can reference an overflow record. But, if we split a * page into multiple chunks, any of the chunks might own any @@ -1688,7 +1689,8 @@ __slvg_row_range_overlap( * discard b_trk. */ if (a_trk->trk_gen > b_trk->trk_gen) { -delete_b: /* +delete_b: + /* * After page and overflow reconciliation, one (and only one) * page can reference an overflow record. But, if we split a * page into multiple chunks, any of the chunks might own any diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index c0a48edf639..1732def4b8d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -149,7 +149,8 @@ __split_verify_root(WT_SESSION_IMPL *session, WT_PAGE *page) return (0); -err: /* Something really bad just happened. */ +err: + /* Something really bad just happened. */ WT_PANIC_RET(session, ret, "fatal error during page split"); } #endif @@ -281,7 +282,8 @@ __split_ref_move(WT_SESSION_IMPL *session, WT_PAGE *from_home, case WT_CELL_ADDR_LEAF_NO: addr->type = WT_ADDR_LEAF_NO; break; - WT_ILLEGAL_VALUE_ERR(session, unpack.raw); + default: + WT_ERR(__wt_illegal_value(session, unpack.raw)); } if (__wt_atomic_cas_ptr(&ref->addr, ref_addr, addr)) addr = NULL; @@ -1601,7 +1603,8 @@ __split_multi_inmem( WT_ERR(__wt_row_modify(session, &cbt, key, NULL, upd, WT_UPDATE_INVALID, true)); break; - WT_ILLEGAL_VALUE_ERR(session, orig->type); + default: + WT_ERR(__wt_illegal_value(session, orig->type)); } } @@ -1629,7 +1632,8 @@ __split_multi_inmem( mod->restore_state = orig->modify->restore_state; FLD_SET(mod->restore_state, WT_PAGE_RS_RESTORED); -err: /* Free any resources that may have been cached in the cursor. */ +err: + /* Free any resources that may have been cached in the cursor. */ WT_TRET(__wt_btcur_close(&cbt, true)); __wt_scr_free(session, &key); diff --git a/src/third_party/wiredtiger/src/btree/bt_stat.c b/src/third_party/wiredtiger/src/btree/bt_stat.c index 70e898a2d3c..d6c535df6b6 100644 --- a/src/third_party/wiredtiger/src/btree/bt_stat.c +++ b/src/third_party/wiredtiger/src/btree/bt_stat.c @@ -130,7 +130,8 @@ __stat_page(WT_SESSION_IMPL *session, WT_PAGE *page, WT_DSRC_STATS **stats) case WT_PAGE_ROW_LEAF: __stat_page_row_leaf(session, page, stats); break; - WT_ILLEGAL_VALUE(session, page->type); + default: + return (__wt_illegal_value(session, page->type)); } return (0); } @@ -245,11 +246,12 @@ __stat_page_row_int( * the in-memory representation of the page doesn't necessarily contain * a reference to the original cell. */ - if (page->dsk != NULL) + if (page->dsk != NULL) { WT_CELL_FOREACH_BEGIN(session, btree, page->dsk, unpack) { if (__wt_cell_type(unpack.cell) == WT_CELL_KEY_OVFL) ++ovfl_cnt; } WT_CELL_FOREACH_END; + } WT_STAT_INCRV(session, stats, btree_overflow, ovfl_cnt); } diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 120ef78fcce..4334b29c7e8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -371,7 +371,8 @@ __wt_sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) WT_CLOCKDIFF_MS(time_stop, time_start)); } -err: /* On error, clear any left-over tree walk. */ +err: + /* On error, clear any left-over tree walk. */ WT_TRET(__wt_page_release(session, walk, flags)); WT_TRET(__wt_page_release(session, prev, flags)); diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index e6740d41a18..ca3a10650d9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -290,7 +290,8 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) } done: -err: /* Inform the underlying block manager we're done. */ +err: + /* Inform the underlying block manager we're done. */ if (bm_start) WT_TRET(bm->verify_end(bm, session)); diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index 4dcf3ed5c30..6f589d0d9ef 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -195,7 +195,8 @@ __wt_verify_dsk_image(WT_SESSION_IMPL *session, const char *tag, case WT_PAGE_BLOCK_MANAGER: case WT_PAGE_OVFL: return (__verify_dsk_chunk(session, tag, dsk, dsk->u.datalen)); - WT_ILLEGAL_VALUE(session, dsk->type); + default: + return (__wt_illegal_value(session, dsk->type)); } /* NOTREACHED */ } @@ -634,7 +635,8 @@ __verify_dsk_row(WT_SESSION_IMPL *session, current->size = prefix + unpack->size; } -key_compare: /* +key_compare: + /* * Compare the current key against the last key. * * Be careful about the 0th key on internal pages: we only store diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index 30cdaf2225a..6f15f918e33 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -353,7 +353,8 @@ __tree_walk_internal(WT_SESSION_IMPL *session, /* If no page is active, begin a walk from the start/end of the tree. */ if ((ref = ref_orig) == NULL) { if (0) { -restart: /* +restart: + /* * Yield before retrying, and if we've yielded enough * times, start sleeping so we don't burn CPU to no * purpose. @@ -467,7 +468,8 @@ restart: /* ++*walkcntp; for (;;) { -descend: /* +descend: + /* * Get a reference, setting the reference hint if it's * wrong (used when we continue the walk). We don't * always update the hints when splitting, it's expected diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index 3fb9803e7ad..3d4c5a52b47 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -258,7 +258,8 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, } if (0) { -err: /* +err: + /* * Remove the update from the current transaction, so we don't * try to modify it on rollback. */ diff --git a/src/third_party/wiredtiger/src/btree/col_srch.c b/src/third_party/wiredtiger/src/btree/col_srch.c index 6d431f05666..6b040557cfc 100644 --- a/src/third_party/wiredtiger/src/btree/col_srch.c +++ b/src/third_party/wiredtiger/src/btree/col_srch.c @@ -117,7 +117,8 @@ __wt_col_search(WT_SESSION_IMPL *session, } if (0) { -restart: /* +restart: + /* * Discard the currently held page and restart the search from * the root. */ @@ -162,7 +163,8 @@ restart: /* base = indx + 1; --limit; } -descend: /* +descend: + /* * Reference the slot used for next step down the tree. * * Base is the smallest index greater than recno and may be the diff --git a/src/third_party/wiredtiger/src/btree/row_key.c b/src/third_party/wiredtiger/src/btree/row_key.c index dfacbe5a435..d849277e43d 100644 --- a/src/third_party/wiredtiger/src/btree/row_key.c +++ b/src/third_party/wiredtiger/src/btree/row_key.c @@ -166,7 +166,8 @@ __wt_row_leaf_key_work(WT_SESSION_IMPL *session, direction = BACKWARD; for (slot_offset = 0;;) { if (0) { -switch_and_jump: /* Switching to a forward roll. */ +switch_and_jump: + /* Switching to a forward roll. */ WT_ASSERT(session, direction == BACKWARD); direction = FORWARD; diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index cf51f1c2e67..9cd2d4df2c3 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -216,7 +216,8 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, } if (0) { -err: /* +err: + /* * Remove the update from the current transaction, so we don't * try to modify it on rollback. */ @@ -312,10 +313,16 @@ __wt_update_obsolete_check( { WT_TXN_GLOBAL *txn_global; WT_UPDATE *first, *next; - u_int count; + uint64_t oldest, stable; + u_int count, upd_seen, upd_unstable; txn_global = &S2C(session)->txn_global; + upd_seen = upd_unstable = 0; + oldest = txn_global->has_oldest_timestamp ? + txn_global->oldest_timestamp : WT_TS_NONE; + stable = txn_global->has_stable_timestamp ? + txn_global->stable_timestamp : WT_TS_NONE; /* * This function identifies obsolete updates, and truncates them from * the rest of the chain; because this routine is called from inside @@ -330,13 +337,28 @@ __wt_update_obsolete_check( for (first = NULL, count = 0; upd != NULL; upd = upd->next, count++) { if (upd->txnid == WT_TXN_ABORTED) continue; - if (!__wt_txn_upd_visible_all(session, upd)) + /* + * If the update isn't visible, it is not obsolete. + */ + ++upd_seen; + if (!__wt_txn_upd_visible_all(session, upd)) { first = NULL; - else if (first == NULL && (WT_UPDATE_DATA_VALUE(upd) || - upd->type == WT_UPDATE_BIRTHMARK)) - first = upd; + /* + * While we're here, also check for the update + * being kept only for timestamp history to + * gauge updates being kept due to history. + */ + if (upd->start_ts != WT_TS_NONE && + upd->start_ts >= oldest && + upd->start_ts < stable) + ++upd_unstable; + } else if (first == NULL && (WT_UPDATE_DATA_VALUE(upd) || + upd->type == WT_UPDATE_BIRTHMARK)) + first = upd; } + __wt_cache_update_lookaside_score(session, upd_seen, upd_unstable); + /* * We cannot discard this WT_UPDATE structure, we can only discard * WT_UPDATE structures subsequent to it, other threads of control will diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index f7ae9164ea5..8603d329c15 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -277,7 +277,8 @@ __wt_row_search(WT_SESSION_IMPL *session, } if (0) { -restart: /* +restart: + /* * Discard the currently held page and restart the search from * the root. */ @@ -431,7 +432,8 @@ append: if (__wt_split_descent_race( goto restart; } -descend: /* Encourage races. */ +descend: + /* Encourage races. */ WT_DIAGNOSTIC_YIELD; /* diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index 6b1a82d0862..78eebe94f12 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -723,7 +723,8 @@ __wt_las_insert_block(WT_CURSOR *cursor, key->size = WT_INSERT_KEY_SIZE(list->ins); } break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } /* @@ -768,7 +769,8 @@ __wt_las_insert_block(WT_CURSOR *cursor, case WT_UPDATE_TOMBSTONE: las_value.size = 0; break; - WT_ILLEGAL_VALUE_ERR(session, upd->type); + default: + WT_ERR(__wt_illegal_value(session, upd->type)); } cursor->set_key(cursor, @@ -817,7 +819,8 @@ __wt_las_insert_block(WT_CURSOR *cursor, "WiredTigerLAS: file size of %" PRIu64 " exceeds maximum " "size %" PRIu64, (uint64_t)las_size, max_las_size); -err: /* Resolve the transaction. */ +err: + /* Resolve the transaction. */ if (local_txn) { if (ret == 0) ret = __wt_txn_commit(session, NULL); diff --git a/src/third_party/wiredtiger/src/checksum/x86/crc32-x86-alt.c b/src/third_party/wiredtiger/src/checksum/x86/crc32-x86-alt.c new file mode 100644 index 00000000000..c7e9f6f0d5b --- /dev/null +++ b/src/third_party/wiredtiger/src/checksum/x86/crc32-x86-alt.c @@ -0,0 +1,100 @@ +/*- + * Public Domain 2014-2019 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <wiredtiger_config.h> +#include <inttypes.h> +#include <stdbool.h> +#include <stddef.h> + +/* + * The hardware-accelerated checksum code that originally shipped on Windows + * did not correctly handle memory that wasn't 8B aligned and a multiple of 8B. + * It's likely that calculations were always 8B aligned, but there's some risk. + * + * What we do is always write the correct checksum, and if a checksum test + * fails, check it against the alternate version have before failing. + */ + +#if defined(_M_AMD64) && !defined(HAVE_NO_CRC32_HARDWARE) +/* + * __checksum_alt -- + * Return a checksum for a chunk of memory, computed in hardware + * using 8 byte steps. + */ +static uint32_t +__checksum_alt(const void *chunk, size_t len) +{ + uint32_t crc; + size_t nqwords; + const uint8_t *p; + const uint64_t *p64; + + crc = 0xffffffff; + + /* Checksum one byte at a time to the first 4B boundary. */ + for (p = chunk; + ((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0 && + len > 0; ++p, --len) { + crc = _mm_crc32_u8(crc, *p); + } + + p64 = (const uint64_t *)p; + /* Checksum in 8B chunks. */ + for (nqwords = len / sizeof(uint64_t); nqwords; nqwords--) { + crc = (uint32_t)_mm_crc32_u64(crc, *p64); + p64++; + } + + /* Checksum trailing bytes one byte at a time. */ + p = (const uint8_t *)p64; + for (len &= 0x7; len > 0; ++p, len--) { + crc = _mm_crc32_u8(crc, *p); + } + + return (~crc); +} + +/* + * __wt_checksum_alt_match -- + * Return if a checksum matches the alternate calculation. + */ +bool +__wt_checksum_alt_match(const void *chunk, size_t len, uint32_t v) +{ + int cpuInfo[4]; + + __cpuid(cpuInfo, 1); + + #define CPUID_ECX_HAS_SSE42 (1 << 20) + if (cpuInfo[2] & CPUID_ECX_HAS_SSE42) + return (__checksum_alt(chunk, len) == v); + + return (false); +} + +#endif diff --git a/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c b/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c index 2d68dcdf63e..207d860c780 100644 --- a/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c +++ b/src/third_party/wiredtiger/src/checksum/x86/crc32-x86.c @@ -27,6 +27,9 @@ */ #include <wiredtiger_config.h> +#if defined(_M_AMD64) +#include <intrin.h> +#endif #include <inttypes.h> #include <stddef.h> diff --git a/src/third_party/wiredtiger/src/config/config.c b/src/third_party/wiredtiger/src/config/config.c index af8344b1257..936e5fd780f 100644 --- a/src/third_party/wiredtiger/src/config/config.c +++ b/src/third_party/wiredtiger/src/config/config.c @@ -557,7 +557,8 @@ __config_process_value(WT_CONFIG_ITEM *value) } if (0) { -nonum: /* +nonum: + /* * We didn't get a well-formed number. That might be okay, the * required type will be checked by __wt_config_check. */ diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 24df42b45f1..d7d27b1c767 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1830,7 +1830,8 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) "option configured"); } -err: /* +err: + /* * We ignore the connection's lock file handle on error, it will be * closed when the connection structure is destroyed. */ @@ -2868,7 +2869,8 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_STATIC_ASSERT(offsetof(WT_CONNECTION_IMPL, iface) == 0); *connectionp = &conn->iface; -err: /* Discard the scratch buffers. */ +err: + /* Discard the scratch buffers. */ __wt_scr_free(session, &encbuf); __wt_scr_free(session, &i1); __wt_scr_free(session, &i2); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 9a279ca3970..ada12f43977 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -356,7 +356,8 @@ __backup_start(WT_SESSION_IMPL *session, WT_ERR(__backup_list_append(session, cb, WT_WIREDTIGER)); } -err: /* Close the hot backup file. */ +err: + /* Close the hot backup file. */ if (srcfs != NULL) WT_TRET(__wt_fclose(session, &srcfs)); /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 508744ed9b4..73225c9613a 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -747,7 +747,8 @@ __curfile_create(WT_SESSION_IMPL *session, WT_STAT_DATA_INCR(session, cursor_create); if (0) { -err: /* +err: + /* * Our caller expects to release the data handle if we fail. * Disconnect it from the cursor before closing. */ @@ -833,7 +834,8 @@ __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, return (0); -err: /* If the cursor could not be opened, release the handle. */ +err: + /* If the cursor could not be opened, release the handle. */ WT_TRET(__wt_session_release_dhandle(session)); return (ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 1a8f6ab28af..07bfe02a142 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -500,7 +500,9 @@ __curjoin_entry_in_range(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, passed = (cmp < 0); break; - WT_ILLEGAL_VALUE(session, WT_CURJOIN_END_RANGE(end)); + default: + return (__wt_illegal_value( + session, WT_CURJOIN_END_RANGE(end))); } if (!passed) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index a34661a7992..5a3ce1268b0 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -52,8 +52,9 @@ static int __json_pack_size(WT_SESSION_IMPL *, const char *, WT_CONFIG_ITEM *, WT_RET(json_string_arg(session, &(jstr), &(pv).u.item));\ (pv).type = 'K'; \ break; \ - /* User format strings have already been validated. */ \ - WT_ILLEGAL_VALUE(session, (pv).type); \ + default: \ + /* User format strings have already been validated. */ \ + return (__wt_illegal_value(session, (pv).type)); \ } \ } while (0) @@ -934,7 +935,8 @@ __wt_json_strncpy(WT_SESSION *wt_session, case '\\': *dst++ = ch; break; - WT_ILLEGAL_VALUE(session, ch); + default: + return (__wt_illegal_value(session, ch)); } else *dst++ = ch; diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index f4aee9a8c90..07092e1d47f 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -711,7 +711,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, /* * We return the statistics field's offset as the key, and a string - * description, a string value, and a uint64_t value as the value + * description, a string value, and a int64_t value as the value * columns. */ cursor->key_format = "i"; diff --git a/src/third_party/wiredtiger/src/evict/evict_file.c b/src/third_party/wiredtiger/src/evict/evict_file.c index 0e806f20608..8257ace107b 100644 --- a/src/third_party/wiredtiger/src/evict/evict_file.c +++ b/src/third_party/wiredtiger/src/evict/evict_file.c @@ -116,7 +116,8 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) } if (0) { -err: /* On error, clear any left-over tree walk. */ +err: + /* On error, clear any left-over tree walk. */ if (next_ref != NULL) WT_TRET(__wt_page_release( session, next_ref, walk_flags)); diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index b0b29942cb2..58555923e37 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -634,10 +634,6 @@ __evict_update_work(WT_SESSION_IMPL *session) LF_ISSET(WT_CACHE_EVICT_CLEAN_HARD)) LF_SET(WT_CACHE_EVICT_DIRTY); - /* When we stop looking for dirty pages, reduce the lookaside score. */ - if (!LF_ISSET(WT_CACHE_EVICT_DIRTY)) - __wt_cache_update_lookaside_score(session, 1, 0); - /* * Scrub dirty pages and keep them in cache if we are less than half * way to the clean or dirty trigger. @@ -2100,7 +2096,8 @@ __evict_walk_tree(WT_SESSION_IMPL *session, !__txn_visible_all_id(session, page->modify->update_txn))) continue; -fast: /* If the page can't be evicted, give up. */ +fast: + /* If the page can't be evicted, give up. */ if (!__wt_page_can_evict(session, ref, NULL)) continue; diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 5df9b7ce9bb..fef5031b6fa 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -243,7 +243,8 @@ err: if (!closing) WT_STAT_DATA_INCR(session, cache_eviction_fail); } -done: /* Leave any local eviction generation. */ +done: + /* Leave any local eviction generation. */ if (local_gen) __wt_session_gen_leave(session, WT_GEN_EVICT); @@ -456,7 +457,8 @@ __evict_page_dirty_update(WT_SESSION_IMPL *session, WT_REF *ref, } break; - WT_ILLEGAL_VALUE(session, mod->rec_result); + default: + return (__wt_illegal_value(session, mod->rec_result)); } return (0); diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index 3f450132f98..cceadbf8c0c 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -1046,7 +1046,8 @@ __cell_data_ref(WT_SESSION_IMPL *session, return (0); huffman = btree->huffman_value; break; - WT_ILLEGAL_VALUE(session, unpack->type); + default: + return (__wt_illegal_value(session, unpack->type)); } return (huffman == NULL || store->size == 0 ? 0 : diff --git a/src/third_party/wiredtiger/src/include/column.i b/src/third_party/wiredtiger/src/include/column.i index a654d73edc0..608d2bffc97 100644 --- a/src/third_party/wiredtiger/src/include/column.i +++ b/src/third_party/wiredtiger/src/include/column.i @@ -13,7 +13,7 @@ static inline WT_INSERT * __col_insert_search_gt(WT_INSERT_HEAD *ins_head, uint64_t recno) { - WT_INSERT *ins, **insp; + WT_INSERT *ins, **insp, *ret_ins; int i; /* If there's no insert chain to search, we're done. */ @@ -28,28 +28,37 @@ __col_insert_search_gt(WT_INSERT_HEAD *ins_head, uint64_t recno) * The insert list is a skip list: start at the highest skip level, then * go as far as possible at each level before stepping down to the next. */ - ins = NULL; - for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0;) - if (*insp != NULL && recno >= WT_INSERT_RECNO(*insp)) { - ins = *insp; /* GTE: keep going at this level */ - insp = &(*insp)->next[i]; + ret_ins = NULL; + for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0;) { + /* + * Use a local variable to access the insert because the skip + * list can change across references. + */ + WT_ORDERED_READ(ins, *insp); + if (ins != NULL && recno >= WT_INSERT_RECNO(ins)) { + /* GTE: keep going at this level */ + insp = &ins->next[i]; + ret_ins = ins; } else { --i; /* LT: drop down a level */ --insp; } + } /* - * If we didn't find any records smaller than the target, we never set - * the return value, set it to the first record in the list. Otherwise, - * it references a record less-than-or-equal to the target, move to a - * later record, that is, a subsequent record greater than the target. - * Because inserts happen concurrently, additional records might be - * inserted after the searched-for record that are still smaller than - * the target, continue to move forward until reaching a record larger - * than the target. There isn't any safety testing because we confirmed - * such a record exists before searching. + * If we didn't find any records greater than or equal to the target, + * we never set the return value, set it to the first record in the + * list. + * + * Otherwise, it references a record less-than-or-equal to the target, + * move to a later record, that is, a subsequent record greater than + * the target. Because inserts happen concurrently, additional records + * might be inserted after the searched-for record that are still + * smaller than the target, continue to move forward until reaching a + * record larger than the target. There isn't any safety testing + * because we confirmed such a record exists before searching. */ - if (ins == NULL) + if ((ins = ret_ins) == NULL) ins = WT_SKIP_FIRST(ins_head); while (recno >= WT_INSERT_RECNO(ins)) ins = WT_SKIP_NEXT(ins); @@ -63,7 +72,7 @@ __col_insert_search_gt(WT_INSERT_HEAD *ins_head, uint64_t recno) static inline WT_INSERT * __col_insert_search_lt(WT_INSERT_HEAD *ins_head, uint64_t recno) { - WT_INSERT *ins, **insp; + WT_INSERT *ins, **insp, *ret_ins; int i; /* If there's no insert chain to search, we're done. */ @@ -78,16 +87,24 @@ __col_insert_search_lt(WT_INSERT_HEAD *ins_head, uint64_t recno) * The insert list is a skip list: start at the highest skip level, then * go as far as possible at each level before stepping down to the next. */ - for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0;) - if (*insp != NULL && recno > WT_INSERT_RECNO(*insp)) { - ins = *insp; /* GT: keep going at this level */ - insp = &(*insp)->next[i]; + ret_ins = NULL; + for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0;) { + /* + * Use a local variable to access the insert because the skip + * list can change across references. + */ + WT_ORDERED_READ(ins, *insp); + if (ins != NULL && recno > WT_INSERT_RECNO(ins)) { + /* GT: keep going at this level */ + insp = &ins->next[i]; + ret_ins = ins; } else { --i; /* LTE: drop down a level */ --insp; } + } - return (ins); + return (ret_ins); } /* @@ -97,38 +114,43 @@ __col_insert_search_lt(WT_INSERT_HEAD *ins_head, uint64_t recno) static inline WT_INSERT * __col_insert_search_match(WT_INSERT_HEAD *ins_head, uint64_t recno) { - WT_INSERT **insp, *ret_ins; + WT_INSERT *ins, **insp; uint64_t ins_recno; int cmp, i; /* If there's no insert chain to search, we're done. */ - if ((ret_ins = WT_SKIP_LAST(ins_head)) == NULL) + if ((ins = WT_SKIP_LAST(ins_head)) == NULL) return (NULL); /* Fast path the check for values at the end of the skiplist. */ - if (recno > WT_INSERT_RECNO(ret_ins)) + if (recno > WT_INSERT_RECNO(ins)) return (NULL); - if (recno == WT_INSERT_RECNO(ret_ins)) - return (ret_ins); + if (recno == WT_INSERT_RECNO(ins)) + return (ins); /* * The insert list is a skip list: start at the highest skip level, then * go as far as possible at each level before stepping down to the next. */ for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0; ) { - if (*insp == NULL) { + /* + * Use a local variable to access the insert because the skip + * list can change across references. + */ + WT_ORDERED_READ(ins, *insp); + if (ins == NULL) { --i; --insp; continue; } - ins_recno = WT_INSERT_RECNO(*insp); + ins_recno = WT_INSERT_RECNO(ins); cmp = (recno == ins_recno) ? 0 : (recno < ins_recno) ? -1 : 1; if (cmp == 0) /* Exact match: return */ - return (*insp); + return (ins); if (cmp > 0) /* Keep going at this level */ - insp = &(*insp)->next[i]; + insp = &ins->next[i]; else { /* Drop down a level */ --i; --insp; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 09d5a780f79..894b47ae3f2 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -21,7 +21,7 @@ struct __wt_process { bool page_version_ts; /* timestamp version page formats */ - /* Checksum function */ + /* Checksum functions */ #define __wt_checksum(chunk, len) __wt_process.checksum(chunk, len) uint32_t (*checksum)(const void *, size_t); diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index 732611491b2..620c581c3f7 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -100,15 +100,6 @@ #define __wt_illegal_value(session, v) \ __wt_illegal_value_func(session, (uintmax_t)(v), __func__, __LINE__) -/* Return and branch-to-err-label cases for switch statements. */ -#define WT_ILLEGAL_VALUE(session, v) \ - default: \ - return (__wt_illegal_value(session, v)) -#define WT_ILLEGAL_VALUE_ERR(session, v) \ - default: \ - ret = __wt_illegal_value(session, v); \ - goto err - #define WT_PANIC_MSG(session, v, ...) do { \ __wt_err(session, v, __VA_ARGS__); \ WT_IGNORE_RET(__wt_panic(session)); \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 1c704be9c3e..aa9ddb0b1fe 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -5,6 +5,7 @@ extern WT_THREAD_RET __wt_cache_pool_server(void *arg) WT_GCC_FUNC_DECL_ATTRIBUT extern WT_UPDATE * __wt_update_obsolete_check( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_block_offset_invalid(WT_BLOCK *block, wt_off_t offset, uint32_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_btree_immediately_durable(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_checksum_alt_match(const void *chunk, size_t len, uint32_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_delete_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool visible_all) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_evict_thread_chk(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_fsync_background_chk(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -907,6 +908,7 @@ static inline bool __wt_btree_lsm_over_size(WT_SESSION_IMPL *session, uint64_t m static inline bool __wt_cache_aggressive(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_cache_full(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_cache_stuck(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline bool __wt_checksum_match(const void *chunk, size_t len, uint32_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_eviction_clean_needed(WT_SESSION_IMPL *session, double *pct_fullp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_eviction_dirty_needed(WT_SESSION_IMPL *session, double *pct_fullp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_eviction_needed( WT_SESSION_IMPL *session, bool busy, bool readonly, double *pct_fullp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index b552f9b1f3e..bd3ef02da54 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -291,3 +291,37 @@ __wt_timing_stress(WT_SESSION_IMPL *session, u_int flag) /* The default maximum delay is 1/10th of a second. */ __wt_sleep(0, i * (WT_TIMING_STRESS_MAX_DELAY / 10)); } + +/* + * The hardware-accelerated checksum code that originally shipped on Windows + * did not correctly handle memory that wasn't 8B aligned and a multiple of 8B. + * It's likely that calculations were always 8B aligned, but there's some risk. + * + * What we do is always write the correct checksum, and if a checksum test + * fails, check it against the alternate version have before failing. + */ + +#if defined(_M_AMD64) && !defined(HAVE_NO_CRC32_HARDWARE) +/* + * __wt_checksum_match -- + * Return if a checksum matches either the primary or alternate values. + */ +static inline bool +__wt_checksum_match(const void *chunk, size_t len, uint32_t v) +{ + return (__wt_checksum(chunk, len) == v || + __wt_checksum_alt_match(chunk, len, v)); +} + +#else + +/* + * __wt_checksum_match -- + * Return if a checksum matches. + */ +static inline bool +__wt_checksum_match(const void *chunk, size_t len, uint32_t v) +{ + return (__wt_checksum(chunk, len) == v); +} +#endif diff --git a/src/third_party/wiredtiger/src/include/packing.i b/src/third_party/wiredtiger/src/include/packing.i index dff8fac64fb..0f47569a4ae 100644 --- a/src/third_party/wiredtiger/src/include/packing.i +++ b/src/third_party/wiredtiger/src/include/packing.i @@ -244,8 +244,9 @@ next: if (pack->cur == pack->end) case 'R': \ (pv).u.u = va_arg(ap, uint64_t); \ break; \ - /* User format strings have already been validated. */ \ - WT_ILLEGAL_VALUE(session, (pv).type); \ + default: \ + /* User format strings have already been validated. */ \ + return (__wt_illegal_value(session, (pv).type)); \ } \ } while (0) @@ -611,8 +612,9 @@ __unpack_read(WT_SESSION_IMPL *session, case 'R': \ *va_arg(ap, uint64_t *) = (pv).u.u; \ break; \ - /* User format strings have already been validated. */ \ - WT_ILLEGAL_VALUE(session, (pv).type); \ + default: \ + /* User format strings have already been validated. */ \ + return (__wt_illegal_value(session, (pv).type)); \ } \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index abb2b1ccc66..ed7ea3b9b67 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -427,7 +427,7 @@ __wt_txn_op_delete_commit_apply_timestamps( /* * __wt_txn_op_set_timestamp -- * Decide whether to copy a commit timestamp into an update. If the op - * structure doesn't have a populated update or ref field or in prepared + * structure doesn't have a populated update or ref field or is in prepared * state there won't be any check for an existing timestamp. */ static inline void @@ -716,7 +716,12 @@ __wt_txn_upd_visible_all(WT_SESSION_IMPL *session, WT_UPDATE *upd) upd->prepare_state == WT_PREPARE_INPROGRESS) return (false); - return (__wt_txn_visible_all(session, upd->txnid, upd->start_ts)); + /* + * This function is used to determine when an update is obsolete: that + * should take into account the durable timestamp which is greater than + * or equal to the start timestamp. + */ + return (__wt_txn_visible_all(session, upd->txnid, upd->durable_ts)); } /* diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 119d734d01f..6d95bc6f94b 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -247,6 +247,9 @@ struct __wt_cursor { * @param cursor the cursor handle * @param ... pointers to hold key fields corresponding to * WT_CURSOR::key_format. + * The API does not validate the argument types passed in, the caller is + * responsible for passing the correct argument types according to + * WT_CURSOR::key_format. * @errors */ int __F(get_key)(WT_CURSOR *cursor, ...); @@ -261,6 +264,9 @@ struct __wt_cursor { * @param cursor the cursor handle * @param ... pointers to hold value fields corresponding to * WT_CURSOR::value_format. + * The API does not validate the argument types passed in, the caller is + * responsible for passing the correct argument types according to + * WT_CURSOR::value_format. * @errors */ int __F(get_value)(WT_CURSOR *cursor, ...); diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index a035a97748b..2534613f32e 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -42,21 +42,21 @@ __wt_log_printf(WT_SESSION_IMPL *session, const char *format, ...) * Given a log record, return whether the checksum matches. */ static bool -__log_checksum_match(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t reclen) +__log_checksum_match(WT_ITEM *buf, uint32_t reclen) { WT_LOG_RECORD *logrec; - uint32_t checksum_calculate, checksum_tmp; + uint32_t checksum_saved, checksum_tmp; + bool checksum_matched; - WT_UNUSED(session); - logrec = (WT_LOG_RECORD *)buf->mem; - checksum_tmp = logrec->checksum; - logrec->checksum = 0; - checksum_calculate = __wt_checksum(logrec, reclen); + logrec = buf->mem; + checksum_saved = checksum_tmp = logrec->checksum; #ifdef WORDS_BIGENDIAN - checksum_calculate = __wt_bswap32(checksum_calculate); + checksum_tmp = __wt_bswap32(checksum_tmp); #endif - logrec->checksum = checksum_tmp; - return (logrec->checksum == checksum_calculate); + logrec->checksum = 0; + checksum_matched = __wt_checksum_match(logrec, reclen, checksum_tmp); + logrec->checksum = checksum_saved; + return (checksum_matched); } /* @@ -1032,7 +1032,7 @@ __log_open_verify(WT_SESSION_IMPL *session, uint32_t id, WT_FH **fhp, goto err; } - if (!__log_checksum_match(session, buf, allocsize)) + if (!__log_checksum_match(buf, allocsize)) WT_ERR_MSG(session, WT_ERROR, "%s: System log record checksum mismatch", fh->name); __wt_log_record_byteswap(logrec); @@ -2479,7 +2479,7 @@ advance: */ buf->size = reclen; logrec = (WT_LOG_RECORD *)buf->mem; - if (!__log_checksum_match(session, buf, reclen)) { + if (!__log_checksum_match(buf, reclen)) { /* * A checksum mismatch means we have reached the end of * the useful part of the log. This should be found on diff --git a/src/third_party/wiredtiger/src/log/log_auto.c b/src/third_party/wiredtiger/src/log/log_auto.c index d7f59fd920e..59a11085f57 100644 --- a/src/third_party/wiredtiger/src/log/log_auto.c +++ b/src/third_party/wiredtiger/src/log/log_auto.c @@ -928,7 +928,8 @@ __wt_txn_op_printlog(WT_SESSION_IMPL *session, WT_RET(__wt_logop_txn_timestamp_print(session, pp, end, args)); break; - WT_ILLEGAL_VALUE(session, optype); + default: + return (__wt_illegal_value(session, optype)); } return (0); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index e24f954478c..0667008485d 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -959,7 +959,8 @@ __clsm_next(WT_CURSOR *cursor) if (clsm->current != NULL) goto retry; } else { -retry: /* +retry: + /* * If there are multiple cursors on that key, move them * forward. */ @@ -1122,7 +1123,8 @@ __clsm_prev(WT_CURSOR *cursor) if (clsm->current != NULL) goto retry; } else { -retry: /* +retry: + /* * If there are multiple cursors on that key, move them * backwards. */ diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index e8838a59606..0479e2a70f4 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -795,7 +795,8 @@ __lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) cookie.chunk_array[i] = NULL; } -err: /* Flush the metadata unless the system is in panic */ +err: + /* Flush the metadata unless the system is in panic */ if (flush_metadata && ret != WT_PANIC) { __wt_lsm_tree_writelock(session, lsm_tree); WT_TRET(__wt_lsm_meta_write(session, lsm_tree, NULL)); diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index f763c49ef2a..8284f44b2cf 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -320,7 +320,8 @@ __wt_meta_track_off(WT_SESSION_IMPL *session, bool need_sync, bool unroll) ret = __wt_checkpoint_sync(session, NULL)); } -err: /* +err: + /* * Undo any tracked operations on failure. * Apply any tracked operations post-commit. */ diff --git a/src/third_party/wiredtiger/src/packing/pack_stream.c b/src/third_party/wiredtiger/src/packing/pack_stream.c index 3850af1ac32..476e33fefc1 100644 --- a/src/third_party/wiredtiger/src/packing/pack_stream.c +++ b/src/third_party/wiredtiger/src/packing/pack_stream.c @@ -95,7 +95,8 @@ wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) WT_RET(__pack_write( session, &pv, &ps->p, (size_t)(ps->end - ps->p))); break; - WT_ILLEGAL_VALUE(session, pv.type); + default: + return (__wt_illegal_value(session, pv.type)); } return (0); @@ -128,7 +129,8 @@ wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i) WT_RET(__pack_write( session, &pv, &ps->p, (size_t)(ps->end - ps->p))); break; - WT_ILLEGAL_VALUE(session, pv.type); + default: + return (__wt_illegal_value(session, pv.type)); } return (0); @@ -158,7 +160,8 @@ wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s) WT_RET(__pack_write( session, &pv, &ps->p, (size_t)(ps->end - ps->p))); break; - WT_ILLEGAL_VALUE(session, pv.type); + default: + return (__wt_illegal_value(session, pv.type)); } return (0); @@ -194,7 +197,8 @@ wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u) WT_RET(__pack_write( session, &pv, &ps->p, (size_t)(ps->end - ps->p))); break; - WT_ILLEGAL_VALUE(session, pv.type); + default: + return (__wt_illegal_value(session, pv.type)); } return (0); @@ -225,7 +229,8 @@ wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) item->data = pv.u.item.data; item->size = pv.u.item.size; break; - WT_ILLEGAL_VALUE(session, pv.type); + default: + return (__wt_illegal_value(session, pv.type)); } return (0); @@ -258,7 +263,8 @@ wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) &pv, (const uint8_t **)&ps->p, (size_t)(ps->end - ps->p))); *ip = pv.u.i; break; - WT_ILLEGAL_VALUE(session, pv.type); + default: + return (__wt_illegal_value(session, pv.type)); } return (0); } @@ -287,7 +293,8 @@ wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp) &pv, (const uint8_t **)&ps->p, (size_t)(ps->end - ps->p))); *sp = pv.u.s; break; - WT_ILLEGAL_VALUE(session, pv.type); + default: + return (__wt_illegal_value(session, pv.type)); } return (0); } @@ -322,7 +329,8 @@ wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) &pv, (const uint8_t **)&ps->p, (size_t)(ps->end - ps->p))); *up = pv.u.u; break; - WT_ILLEGAL_VALUE(session, pv.type); + default: + return (__wt_illegal_value(session, pv.type)); } return (0); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_child.c b/src/third_party/wiredtiger/src/reconcile/rec_child.c index f1d261c8f42..66034c0a8df 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_child.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_child.c @@ -289,7 +289,9 @@ __wt_rec_child_modify(WT_SESSION_IMPL *session, WT_ASSERT(session, WT_REF_SPLIT != WT_REF_SPLIT); return (__wt_set_return(session, EBUSY)); - WT_ILLEGAL_VALUE(session, r->tested_ref_state); + default: + return (__wt_illegal_value( + session, r->tested_ref_state)); } WT_STAT_CONN_INCR(session, child_modify_blocked_page); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 998d6eb2aa6..cbc36435db3 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -272,8 +272,9 @@ __wt_rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) case WT_PM_REC_REPLACE: addr = &child->modify->mod_replace; break; - WT_ILLEGAL_VALUE_ERR( - session, child->modify->rec_result); + default: + WT_ERR(__wt_illegal_value( + session, child->modify->rec_result)); } break; case WT_CHILD_ORIGINAL: @@ -802,7 +803,8 @@ __wt_rec_col_var(WT_SESSION_IMPL *session, WT_ERR(__wt_dsk_cell_data_ref( session, WT_PAGE_COL_VAR, vpack, orig)); -record_loop: /* +record_loop: + /* * Generate on-page entries: loop repeat records, looking for * WT_INSERT entries matching the record number. The WT_INSERT * lists are in sorted order, so only need check the next one. @@ -864,7 +866,9 @@ record_loop: /* case WT_UPDATE_TOMBSTONE: deleted = true; break; - WT_ILLEGAL_VALUE_ERR(session, upd->type); + default: + WT_ERR(__wt_illegal_value( + session, upd->type)); } } else if (vpack->raw == WT_CELL_VALUE_OVFL_RM) { /* @@ -972,7 +976,8 @@ record_loop: /* } } -compare: /* +compare: + /* * If we have a record against which to compare, and * the records compare equal, increment the rle counter * and continue. If the records don't compare equal, @@ -1172,7 +1177,9 @@ compare: /* case WT_UPDATE_TOMBSTONE: deleted = true; break; - WT_ILLEGAL_VALUE_ERR(session, upd->type); + default: + WT_ERR(__wt_illegal_value( + session, upd->type)); } } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index b234bb0641e..005dfb20776 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -456,8 +456,9 @@ __wt_rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) */ addr = &child->modify->mod_replace; break; - WT_ILLEGAL_VALUE_ERR( - session, child->modify->rec_result); + default: + WT_ERR(__wt_illegal_value( + session, child->modify->rec_result)); } break; case WT_CHILD_ORIGINAL: @@ -657,7 +658,8 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) break; case WT_UPDATE_TOMBSTONE: continue; - WT_ILLEGAL_VALUE(session, upd->type); + default: + return (__wt_illegal_value(session, upd->type)); } /* Build key cell. */ @@ -976,7 +978,8 @@ __wt_rec_row_leaf(WT_SESSION_IMPL *session, /* Proceed with appended key/value pairs. */ goto leaf_insert; - WT_ILLEGAL_VALUE_ERR(session, upd->type); + default: + WT_ERR(__wt_illegal_value(session, upd->type)); } } @@ -1099,7 +1102,8 @@ build: /* Update compression state. */ __rec_key_state_update(r, ovfl_key); -leaf_insert: /* Write any K/V pairs inserted into the page after this key. */ +leaf_insert: + /* Write any K/V pairs inserted into the page after this key. */ if ((ins = WT_SKIP_FIRST(WT_ROW_INSERT(page, rip))) != NULL) WT_ERR(__rec_row_leaf_insert(session, r, ins)); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 9222272767b..93eea9a5fb5 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -182,11 +182,9 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, if (upd->prepare_state == WT_PREPARE_LOCKED || upd->prepare_state == WT_PREPARE_INPROGRESS) prepared = true; - else if ((F_ISSET(r, WT_REC_VISIBLE_ALL) ? + else if (F_ISSET(r, WT_REC_VISIBLE_ALL) ? WT_TXNID_LE(r->last_running, txnid) : - !__txn_visible_id(session, txnid)) || - (upd->start_ts != WT_TS_NONE && - !__wt_txn_upd_durable(session, upd))) + !__txn_visible_id(session, txnid)) uncommitted = r->update_uncommitted = true; if (prepared || uncommitted) @@ -198,12 +196,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, first_ts_upd = upd; /* - * Find the first update we can use. - * - * Update/restore eviction can handle any update (including - * uncommitted updates). Lookaside eviction can save any - * committed update. Regular eviction checks that the maximum - * transaction ID and timestamp seen are stable. + * Select the update to write to the disk image. * * Lookaside and update/restore eviction try to choose the same * version as a subsequent checkpoint, so that checkpoint can @@ -218,9 +211,8 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, if (upd_select->upd == NULL && r->las_skew_newest) upd_select->upd = upd; - if ((F_ISSET(r, WT_REC_VISIBLE_ALL) ? + if (F_ISSET(r, WT_REC_VISIBLE_ALL) ? !__wt_txn_upd_visible_all(session, upd) : - !__wt_txn_upd_visible(session, upd)) || !__wt_txn_upd_durable(session, upd)) { if (F_ISSET(r, WT_REC_EVICT)) ++r->updates_unstable; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 5a78f0df88d..f7eeada4c8e 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -500,7 +500,8 @@ __rec_root_write(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) return (0); case WT_PM_REC_MULTIBLOCK: /* Multiple blocks */ break; - WT_ILLEGAL_VALUE(session, mod->rec_result); + default: + return (__wt_illegal_value(session, mod->rec_result)); } __wt_verbose(session, WT_VERB_SPLIT, @@ -1401,7 +1402,8 @@ __wt_rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) r->min_space_avail = r->min_split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); -done: /* +done: + /* * Overflow values can be larger than the maximum page size but still be * "on-page". If the next key/value pair is larger than space available * after a split has happened (in other words, larger than the maximum @@ -2023,7 +2025,8 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, case WT_PAGE_ROW_INT: multi->addr.type = WT_ADDR_INT; break; - WT_ILLEGAL_VALUE(session, page->type); + default: + return (__wt_illegal_value(session, page->type)); } multi->size = WT_STORE_SIZE(chunk->image.size); multi->checksum = 0; @@ -2412,7 +2415,8 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) mod->mod_replace.size = 0; __wt_free(session, mod->mod_disk_image); break; - WT_ILLEGAL_VALUE(session, mod->rec_result); + default: + return (__wt_illegal_value(session, mod->rec_result)); } /* Reset the reconciliation state. */ diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 7db62d29738..86f988ac63e 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -1481,7 +1481,8 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, WT_ERR(__wt_schema_range_truncate(session, start, stop)); done: -err: /* +err: + /* * Close any locally-opened start cursor. * * Reset application cursors, they've possibly moved and the diff --git a/src/third_party/wiredtiger/src/support/global.c b/src/third_party/wiredtiger/src/support/global.c index ec1df0e8a91..3cbfe81ee46 100644 --- a/src/third_party/wiredtiger/src/support/global.c +++ b/src/third_party/wiredtiger/src/support/global.c @@ -119,6 +119,10 @@ __global_once(void) TAILQ_INIT(&__wt_process.connqh); + /* + * Set up the checksum functions. If there's only one, set it as the + * alternate, that way code doesn't have to check if it's set or not. + */ __wt_process.checksum = wiredtiger_crc32c_func(); __global_calibrate_ticks(); diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index a44f349e3e5..1093408cafe 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -378,7 +378,8 @@ __wt_hazard_check(WT_SESSION_IMPL *session, WT_STAT_CONN_INCRV(session, cache_hazard_walks, walk_cnt); hp = NULL; -done: /* Leave the current resource generation. */ +done: + /* Leave the current resource generation. */ __wt_session_gen_leave(session, WT_GEN_HAZARD); return (hp); diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index a81be9aaad2..5fca8e12eb4 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -179,7 +179,8 @@ __wt_buf_set_printable_format(WT_SESSION_IMPL *session, session, buf, "%s%" PRIu64, sep, pv.u.u)); sep = ","; break; - WT_ILLEGAL_VALUE_ERR(session, pv.type); + default: + WT_ERR(__wt_illegal_value(session, pv.type)); } } WT_ERR_NOTFOUND_OK(ret); diff --git a/src/third_party/wiredtiger/src/support/thread_group.c b/src/third_party/wiredtiger/src/support/thread_group.c index 1bb4df79b0d..a798c02fbf2 100644 --- a/src/third_party/wiredtiger/src/support/thread_group.c +++ b/src/third_party/wiredtiger/src/support/thread_group.c @@ -225,7 +225,8 @@ __thread_group_resize( __wt_thread_group_start_one(session, group, true); return (0); -err: /* +err: + /* * An error resizing a thread array is currently fatal, it should only * happen in an out of memory situation. Do real cleanup just in case * that changes in the future. diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index f7c94399710..3228a70ca5e 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -1078,7 +1078,8 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) __wt_cache_eviction_check(session, false, false, NULL)); return (0); -err: /* +err: + /* * If anything went wrong, roll back. * * !!! @@ -1612,7 +1613,7 @@ int __wt_verbose_dump_txn_one(WT_SESSION_IMPL *session, WT_TXN *txn) { const char *iso_tag; - char ts_string[4][WT_TS_INT_STRING_SIZE]; + char ts_string[5][WT_TS_INT_STRING_SIZE]; WT_NOT_READ(iso_tag, "INVALID"); switch (txn->isolation) { @@ -1627,22 +1628,34 @@ __wt_verbose_dump_txn_one(WT_SESSION_IMPL *session, WT_TXN *txn) break; } WT_RET(__wt_msg(session, - "mod count: %u" + "transaction id: %" PRIu64 + ", mod count: %u" ", snap min: %" PRIu64 ", snap max: %" PRIu64 + ", snapshot count: %u" ", commit_timestamp: %s" ", durable_timestamp: %s" ", first_commit_timestamp: %s" + ", prepare_timestamp: %s" ", read_timestamp: %s" + ", checkpoint LSN: [%" PRIu32 "][%" PRIu32 "]" + ", full checkpoint: %s" + ", rollback reason: %s" ", flags: 0x%08" PRIx32 ", isolation: %s", + txn->id, txn->mod_count, txn->snap_min, txn->snap_max, + txn->snapshot_count, __wt_timestamp_to_string(txn->commit_timestamp, ts_string[0]), __wt_timestamp_to_string(txn->durable_timestamp, ts_string[1]), __wt_timestamp_to_string(txn->first_commit_timestamp, ts_string[2]), - __wt_timestamp_to_string(txn->read_timestamp, ts_string[3]), + __wt_timestamp_to_string(txn->prepare_timestamp, ts_string[3]), + __wt_timestamp_to_string(txn->read_timestamp, ts_string[4]), + txn->ckpt_lsn.l.file, txn->ckpt_lsn.l.offset, + txn->full_ckpt ? "true" : "false", + txn->rollback_reason == NULL ? "" : txn->rollback_reason, txn->flags, iso_tag)); return (0); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 0b4e9394f9f..5d76fc4bd56 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1012,7 +1012,8 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) conn->txn_global.last_ckpt_timestamp = ckpt_tmp_ts; } -err: /* +err: + /* * Reset the timer so that next checkpoint tracks the progress only if * configured. */ @@ -1678,7 +1679,8 @@ __checkpoint_tree( else WT_ERR(__wt_evict_file(session, WT_SYNC_CLOSE)); -fake: /* +fake: + /* * If we're faking a checkpoint and logging is enabled, recovery should * roll forward any changes made between now and the next checkpoint, * so set the checkpoint LSN to the beginning of time. @@ -1727,7 +1729,8 @@ fake: /* WT_ERR(__wt_txn_checkpoint_log( session, false, WT_TXN_LOG_CKPT_STOP, NULL)); -err: /* Resolved the checkpoint for the block manager in the error path. */ +err: + /* Resolved the checkpoint for the block manager in the error path. */ if (resolve_bm) WT_TRET(bm->checkpoint_resolve(bm, session, ret != 0)); diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index b9d9c26d941..07a1b1152cb 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -94,7 +94,8 @@ __txn_op_log(WT_SESSION_IMPL *session, WT_ITEM *logrec, WT_RET(__wt_logop_row_remove_pack( session, logrec, fileid, &cursor->key)); break; - WT_ILLEGAL_VALUE(session, upd->type); + default: + return (__wt_illegal_value(session, upd->type)); } } else { recno = WT_INSERT_RECNO(cbt->ins); @@ -113,7 +114,8 @@ __txn_op_log(WT_SESSION_IMPL *session, WT_ITEM *logrec, WT_RET(__wt_logop_col_remove_pack( session, logrec, fileid, recno)); break; - WT_ILLEGAL_VALUE(session, upd->type); + default: + return (__wt_illegal_value(session, upd->type)); } } @@ -574,7 +576,8 @@ __wt_txn_checkpoint_log( __wt_scr_free(session, &txn->ckpt_snapshot); txn->full_ckpt = false; break; - WT_ILLEGAL_VALUE_ERR(session, flags); + default: + WT_ERR(__wt_illegal_value(session, flags)); } err: __wt_logrec_free(session, &logrec); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index ff4c2b8b9cb..550d96fd27f 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -265,8 +265,8 @@ __txn_op_apply( case WT_TXN_TRUNC_STOP: stop = cursor; break; - - WT_ILLEGAL_VALUE_ERR(session, mode); + default: + WT_ERR(__wt_illegal_value(session, mode)); } /* Set the keys. */ @@ -291,8 +291,8 @@ __txn_op_apply( WT_ERR(__wt_logop_txn_timestamp_unpack(session, pp, end, &t_sec, &t_nsec, &commit, &durable, &first, &prepare, &read)); break; - - WT_ILLEGAL_VALUE_ERR(session, optype); + default: + WT_ERR(__wt_illegal_value(session, optype)); } done: diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 1779a48774a..01dad40f85f 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -327,7 +327,8 @@ __txn_abort_newer_updates( case WT_PAGE_ROW_LEAF: __txn_abort_newer_row_leaf(session, page, rollback_timestamp); break; - WT_ILLEGAL_VALUE_ERR(session, page->type); + default: + WT_ERR(__wt_illegal_value(session, page->type)); } err: if (local_read) diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index 879e0a68a44..ff2c3883796 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -147,7 +147,8 @@ load_dump(WT_SESSION *session) } else ret = insert(cursor, uri); -err: /* +err: + /* * Technically, we don't have to close the cursor because the session * handle will do it for us, but I'd like to see the flush to disk and * the close succeed, it's better to fail early when loading files. diff --git a/src/third_party/wiredtiger/test/suite/test_las01.py b/src/third_party/wiredtiger/test/suite/test_las01.py index 6f736d72238..76f19b51768 100755 --- a/src/third_party/wiredtiger/test/suite/test_las01.py +++ b/src/third_party/wiredtiger/test/suite/test_las01.py @@ -99,7 +99,7 @@ class test_las01(wttest.WiredTigerTestCase): ds.populate() bigvalue = b"aaaaa" * 100 - # Initially load huge data + # Initially load huge data. cursor = self.session.open_cursor(uri) for i in range(1, 10000): cursor.set_key(ds.key(nrows + i)) @@ -109,44 +109,44 @@ class test_las01(wttest.WiredTigerTestCase): self.session.checkpoint() # Scenario: 1 - # Check to see LAS working with old snapshot + # Check to see if LAS is working with the old snapshot. bigvalue1 = b"bbbbb" * 100 self.session.snapshot("name=xxx") - # Update the values in different session after snapshot + # Update the values in different session after snapshot. self.large_updates(self.session, uri, bigvalue1, ds, nrows) - # Check to see the value after recovery + # Check to see the value after recovery. self.durable_check(bigvalue1, uri, ds, nrows) self.session.snapshot("drop=(all)") # Scenario: 2 - # Check to see LAS working with old reader + # Check to see if LAS is working with the old reader. bigvalue2 = b"ccccc" * 100 session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') self.large_updates(self.session, uri, bigvalue2, ds, nrows) - # Check to see the value after recovery + # Check to see the value after recovery. self.durable_check(bigvalue2, uri, ds, nrows) session2.rollback_transaction() session2.close() # Scenario: 3 - # Check to see LAS working with modify operations + # Check to see LAS working with modify operations. bigvalue3 = b"ccccc" * 100 bigvalue3 = b'AA' + bigvalue3[2:] session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') - # Apply two modify operations - replacing the first two items with 'A' + # Apply two modify operations - replacing the first two items with 'A'. self.session.begin_transaction() self.large_modifies(self.session, uri, 0, ds, nrows) self.large_modifies(self.session, uri, 1, ds, nrows) self.session.commit_transaction() - # Check to see the value after recovery + # Check to see the value after recovery. self.durable_check(bigvalue3, uri, ds, nrows) session2.rollback_transaction() session2.close() # Scenario: 4 - # Check to see LAS working with old timestamp + # Check to see if LAS is working with the old timestamp. bigvalue4 = b"ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) self.large_updates(self.session, uri, bigvalue4, ds, nrows, timestamp=True) @@ -154,7 +154,7 @@ class test_las01(wttest.WiredTigerTestCase): self.durable_check(bigvalue3, uri, ds, nrows) self.conn.set_timestamp('stable_timestamp=' + timestamp_str(i + 1)) - # Check to see latest data can be seen + # Check that the latest data can be seen. self.durable_check(bigvalue4, uri, ds, nrows) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_las03.py b/src/third_party/wiredtiger/test/suite/test_las03.py index cf327a19e1e..459f7fb4a5d 100755 --- a/src/third_party/wiredtiger/test/suite/test_las03.py +++ b/src/third_party/wiredtiger/test/suite/test_las03.py @@ -65,20 +65,20 @@ class test_las03(wttest.WiredTigerTestCase): ds.populate() bigvalue = b"aaaaa" * 100 - # Initially load huge data + # Initially load huge data. cursor = self.session.open_cursor(uri) for i in range(1, 10000): cursor[ds.key(nrows + i)] = bigvalue cursor.close() self.session.checkpoint() - # Check to see LAS working with old timestamp + # Check to see LAS working with old timestamp. bigvalue2 = b"ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) las_writes_start = self.get_stat(stat.conn.cache_write_lookaside) self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000) - # If the test sizing is correct, the history will overflow the cache + # If the test sizing is correct, the history will overflow the cache. self.session.checkpoint() las_writes = self.get_stat(stat.conn.cache_write_lookaside) - las_writes_start self.assertGreaterEqual(las_writes, 0) @@ -86,7 +86,7 @@ class test_las03(wttest.WiredTigerTestCase): for ts in range(2, 4): self.conn.set_timestamp('stable_timestamp=' + timestamp_str(ts)) - # Now just update one record and checkpoint again + # Now just update one record and checkpoint again. self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1) las_reads_start = self.get_stat(stat.conn.cache_read_lookaside) diff --git a/src/third_party/wiredtiger/test/suite/test_las05.py b/src/third_party/wiredtiger/test/suite/test_las05.py new file mode 100755 index 00000000000..2011aed4dc6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_las05.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import copy_wiredtiger_home +import wiredtiger, wttest +from wiredtiger import stat +from wtdataset import SimpleDataSet + +def timestamp_str(t): + return '%x' % t + +# test_las05.py +# Verify lookaside_score reflects cache pressure due to history +# even if we're not yet actively pushing into the lookaside file. +class test_las05(wttest.WiredTigerTestCase): + # Force a small cache. + conn_config = 'cache_size=50MB,statistics=(fast)' + session_config = 'isolation=snapshot' + stable = 1 + + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + + def large_updates(self, session, uri, value, ds, nrows, nops): + # Update a large number of records, we'll hang if the lookaside table + # isn't doing its thing. + cursor = session.open_cursor(uri) + score_start = self.get_stat(stat.conn.cache_lookaside_score) + for i in range(nrows + 1, nrows + nops + 1): + session.begin_transaction() + cursor[ds.key(i)] = value + session.commit_transaction('commit_timestamp=' + timestamp_str(self.stable + i)) + cursor.close() + score_end = self.get_stat(stat.conn.cache_lookaside_score) + score_diff = score_end - score_start + self.pr("After large updates score start: " + str(score_start)) + self.pr("After large updates score end: " + str(score_end)) + self.pr("After large updates lookaside score diff: " + str(score_diff)) + + def test_checkpoint_las_reads(self): + # Create a small table. + uri = "table:test_las05" + nrows = 100 + ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds.populate() + bigvalue = b"aaaaa" * 100 + + # Initially load huge data. + # Add 10000 items that have a 500b value that is about 50Mb that + # is the entire cache. Then checkpoint so that none is required + # to stay in cache. + cursor = self.session.open_cursor(uri) + for i in range(1, 10000): + cursor[ds.key(nrows + i)] = bigvalue + cursor.close() + self.session.checkpoint() + + # Pin the oldest timestamp so that all history has to stay. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + # Loop a couple times, partly filling the cache but not + # overfilling it to see the lookaside score value change + # even if lookaside is not yet in use. + # + # Use smaller values, 50 bytes and fill 8 times, under full cache. + valstr='abcdefghijklmnopqrstuvwxyz' + loop_start = self.get_stat(stat.conn.cache_lookaside_score) + for i in range(1, 9): + bigvalue2 = valstr[i].encode() * 50 + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) + entries_start = self.get_stat(stat.conn.cache_lookaside_entries) + score_start = self.get_stat(stat.conn.cache_lookaside_score) + self.pr("Update iteration: " + str(i) + " Value: " + str(bigvalue2)) + self.pr("Update iteration: " + str(i) + " Score: " + str(score_start)) + self.large_updates(self.session, uri, bigvalue2, ds, nrows, nrows) + self.stable += nrows + score_end = self.get_stat(stat.conn.cache_lookaside_score) + entries_end = self.get_stat(stat.conn.cache_lookaside_entries) + # We expect to see the lookaside score increase but not writing + # any new entries to lookaside. + self.assertGreaterEqual(score_end, score_start) + self.assertEqual(entries_end, entries_start) + + # While each iteration may or may not increase the score, we expect the + # score to have strictly increased from before the loop started. + loop_end = self.get_stat(stat.conn.cache_lookaside_score) + self.assertGreater(loop_end, loop_start) + + # Now move oldest timestamp forward and insert a couple large updates + # but we should see the score drop because we allowed the history to move. + # By moving the oldest after updating we should see the score drop + # to zero. + score_start = loop_end + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(self.stable)) + for i in range(9, 11): + bigvalue2 = valstr[i].encode() * 50 + self.pr("Update iteration with oldest: " + str(i) + " Value: " + str(bigvalue2)) + self.large_updates(self.session, uri, bigvalue2, ds, nrows, nrows) + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(self.stable)) + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(self.stable)) + self.stable += nrows + score_end = self.get_stat(stat.conn.cache_lookaside_score) + self.assertLess(score_end, score_start) + self.assertEqual(score_end, 0) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare07.py b/src/third_party/wiredtiger/test/suite/test_prepare07.py new file mode 100644 index 00000000000..d021531df69 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare07.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, shutil, time +from helper import copy_wiredtiger_home +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +def timestamp_str(t): + return '%x' % t + +# test_prepare07.py +# Test to ensure prepared updates older than oldest timestamp are not visible. +# this test is mainly to ensure there is no gap in txn_visible_all when active +# oldest transaction is a prepared transaction, and the oldest timestamp advanced +# past the prepared timestamp. +class test_prepare07(wttest.WiredTigerTestCase): + # Force a small cache. + conn_config = 'cache_size=50MB' + + def older_prepare_updates(self, uri, ds, nrows, value_a): + # Commit some updates along with a prepared update, which is not resolved. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100)) + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + + # Commit some updates. + value_b = b"bbbbb" * 100 + cursor = self.session.open_cursor(uri) + self.session.begin_transaction('isolation=snapshot') + cursor.set_key(ds.key(nrows + 1)) + cursor.set_value(value_b) + self.assertEquals(cursor.update(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(110)) + self.session.begin_transaction('isolation=snapshot') + cursor.set_key(ds.key(nrows + 2)) + cursor.set_value(value_b) + self.assertEquals(cursor.update(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(120)) + + # Prepare a transaction and keep it open. + session_p = self.conn.open_session() + cursor_p = session_p.open_cursor(uri) + session_p.begin_transaction('isolation=snapshot') + cursor_p.set_key(ds.key(nrows + 3)) + cursor_p.set_value(value_b) + self.assertEquals(cursor_p.update(), 0) + session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(130)) + + # Commit some more updates. + self.session.begin_transaction('isolation=snapshot') + cursor.set_key(ds.key(nrows + 4)) + cursor.set_value(value_b) + self.assertEquals(cursor.update(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(140)) + self.session.begin_transaction('isolation=snapshot') + cursor.set_key(ds.key(nrows + 5)) + cursor.set_value(value_b) + self.assertEquals(cursor.update(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(150)) + + # Move the oldest and the stable timestamp to the latest. + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(155)) + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(155)) + + # Commit an update newer than the stable timestamp. + self.session.begin_transaction('isolation=snapshot') + cursor.set_key(ds.key(nrows + 6)) + cursor.set_value(value_b) + self.assertEquals(cursor.update(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(160)) + + # Take a checkpoint here, so that prepared transaction will not be durable.. + self.session.checkpoint() + + # Commit the prepared transaction. + session_p.commit_transaction('commit_timestamp=' + timestamp_str(140) + ',durable_timestamp=' + timestamp_str(160)) + + # Take a backup and check the values. + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.backup(self.backup_dir) + backup_conn = self.wiredtiger_open(self.backup_dir) + session_b = backup_conn.open_session() + cursor_b = session_b.open_cursor(uri) + + # Committed non - prepared transactions data should be seen. + cursor_b.set_key(ds.key(nrows + 1)) + self.assertEquals(cursor_b.search(), 0) + self.assertEquals(cursor_b.get_value(),value_b) + cursor_b.set_key(ds.key(nrows + 2)) + self.assertEquals(cursor_b.search(), 0) + self.assertEquals(cursor_b.get_value(),value_b) + # Committed prepared transaction data should not be seen. + cursor_b.set_key(ds.key(nrows + 3)) + self.assertEquals(cursor_b.search(), 0) + self.assertEquals(cursor_b.get_value(),value_a) + # Committed non - prepared transactions data should be seen. + cursor_b.set_key(ds.key(nrows + 4)) + self.assertEquals(cursor_b.search(), 0) + self.assertEquals(cursor_b.get_value(),value_b) + cursor_b.set_key(ds.key(nrows + 5)) + self.assertEquals(cursor_b.search(), 0) + self.assertEquals(cursor_b.get_value(),value_b) + + # Committed transactions newer to the stable timestamp should not be seen. + cursor_b.set_key(ds.key(nrows + 6)) + self.assertEquals(cursor_b.search(), 0) + self.assertEquals(cursor_b.get_value(),value_a) + + # close sessions. + cursor_p.close() + session_p.close() + cursor.close() + self.session.close() + cursor_b.close() + session_b.close() + + def test_older_prepare_updates(self): + # Create a small table. + uri = "table:test" + nrows = 100 + ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') + ds.populate() + value_a = b"aaaaa" * 100 + + # Initially load huge data + cursor = self.session.open_cursor(uri) + for i in range(1, 10000): + cursor.set_key(ds.key(nrows + i)) + cursor.set_value(value_a) + self.assertEquals(cursor.insert(), 0) + cursor.close() + self.session.checkpoint() + + # Check if txn_visible_all is working properly, when an active oldest + # transaction is a prepared transaction and the oldest timestamp + # advances beyond the prepared timestamp. + self.older_prepare_updates(uri, ds, nrows, value_a) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index de9514b7651..5fbcecac105 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -560,12 +560,13 @@ class WiredTigerTestCase(unittest.TestCase): def raisesBusy(self, expr): """ - Execute the expression, returning true if a 'Resource busy' - exception is raised, returning false if no exception is raised. + Execute the expression, returning true if a 'Resource busy' exception + is raised, returning false if no exception is raised. Some systems + report 'Device or resource busy', allow either. Any other exception raises a test suite failure. """ return self.assertRaisesException(wiredtiger.WiredTigerError, \ - expr, exceptionString='Resource busy', optional=True) + expr, exceptionString='/[Rr]esource busy/', optional=True) def assertTimestampsEqual(self, ts1, ts2): """ |