diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-09-23 17:19:39 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-09-23 07:55:26 +0000 |
| commit | 5422fde7665a292d3fe69e3198d43edc3a2366a2 (patch) | |
| tree | 5cdef69257a33e0790979c3be501ab0cd851d73f | |
| parent | 3e22135e6be46b70ac466c3783fbcf3cc0084ded (diff) | |
| download | mongo-5422fde7665a292d3fe69e3198d43edc3a2366a2.tar.gz | |
Import wiredtiger: a9bd69a7ca5dd14ae36ab76574ac7eb1c0207758 from branch mongodb-4.4
ref: 9893156289..a9bd69a7ca
for: 4.4.10
WT-6193 Re-enable VLCS testing in format-test
WT-6669 Enable VLCS coverage and checkpoint tests in evergreen
WT-6900 Write "schema" subpage for Architecture Guide
WT-6903 Write "dhandle/btree" subpage for Architecture Guide
WT-6907 Write "snapshots" subpage for Architecture Guide
WT-6909 Eviction architecture guide
WT-6913 file system and os interface architecture guide
WT-7059 Upgrade Evergreen distros to Ubuntu 20.04
WT-7294 Re-enable VLCS evergreen endianness tests
WT-7601 Fix typo in operation tracking documentation
WT-7695 Dump entire tree when key out-of-order found in __cursor_key_order_check_row
WT-7745 Add macro to identify uris for btree objects
WT-7844 Add tiered_abort stress test for tiered storage.
WT-7902 Retry the alter command after a system wide checkpoint
WT-7914 Update the documentation only when required
WT-7942 Release timestamp lock in test/format when all_durable timestamp is not found
WT-7949 Change local store cache and bucket directories to be relative to WT_HOME
WT-7957 Tiered storage should look in local system for objects
WT-7959 Skipped_pages is less than expected_pages_skipped in test_cursor17
WT-7987 Create Unit Test to check compact does not rewrite overflow items
WT-7989 Compact quits when running at the same time as a system checkpoint
WT-7993 If gathering handles and not in aggressive eviction mode, have eviction sleep to let checkpoint grab a contentious spinlock.
WT-8000 Allow s_docs to compile even if doxygen version is not supported
WT-8001 Fix Inconsistent API behaviour when setting global oldest and stable timestamps
WT-8007 Update script to correctly generate new test for the CPP test suite framework
WT-8011 Add format support for randomly choosing RS or VLCS
WT-8017 Re-enable VLCS format stress testing in evergreen.
WT-8019 VLCS snapshot-isolation search mismatch
WT-8023 Use global transaction information to asses if a session has active transaction
WT-8024 Add link text to cross-references in Arch Guide
WT-8034 Use the PR branch when docs are compiled in PR testing
WT-8035 Handle prefix enabled lexicographical comparisons for string key formats
WT-8039 Add a new flag for API check instead of clearing prepared flag which may incorrectly force a roll back
WT-8041 Rollback to stable unpacks values repeatedly
WT-8042 Create an evergreen job to run test/checkpoint variation
WT-8043 Tidy the "ripcip" argument into the visibility code
WT-8044 Prefix enabled search near only returns matching keys
WT-8050 Run large-scale-tests on ubuntu-2004
WT-8057 Add a test to verify that changing compact is not resulting in data loss
WT-8075 Coverity analysis defect 120712: 'Constant' variable guards dead code
WT-8077 Mark the page dirty once the prepared updates are resolved
WT-8079 Add breakpoint to verify error macros, clean up API processing
WT-8081 Fix tiered hook functions to provide default for config string
WT-8093 Improve the CPP testing framework for better flexibility
WT-8094 Fix use after free in csv extractor
WT-8103 Skip a dhandle if it isn't a btree
WT-8104 Fix memory leaks noted by ASAN
WT-8108 Use temporary files and rename in local store
WT-8112 Skip null check, per coverity
WT-8113 Remove dead code, per coverity
WT-8121 Create a long running stress test which inserts a large amount of data over a long period
WT-8125 Update hs_cleanup cppsuite test to use the new thread_context logic
WT-8126 Mark btree as dirty only if not newly created when instantiating a deleted row-store leaf page
WT-8148 Fix comment typo in util_verify.c
121 files changed, 5270 insertions, 1662 deletions
diff --git a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 index b912335fd16..545ca02a451 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 @@ -138,6 +138,11 @@ AC_DEFUN([AM_CLANG_WARNINGS], [ # conditional body. w="$w -Wno-extra-semi-stmt" + # FIXME-WT-8052: Figure out whether we want to disable these or change the code. + w="$w -Wno-implicit-int-float-conversion" + w="$w -Wno-implicit-fallthrough" + w="$w -Wno-maybe-uninitialized" + # Ignore unrecognized options. w="$w -Wno-unknown-warning-option" diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index ca9d3f2ed6e..70307efdd2e 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -218,6 +218,10 @@ tiered_config = [ the bucket indicating the location for this table'''), Config('bucket_prefix', '', r''' the unique bucket prefix for this table'''), + Config('cache_directory', '', r''' + a directory to store locally cached versions of files in the storage source. By + default, it is named with \c "-cache" appended to the bucket name. A relative + directory name is relative to the home directory'''), Config('local_retention', '300', r''' time in seconds to retain data on tiered storage on the local tier for faster read access''', @@ -234,6 +238,10 @@ tiered_tree_config = [ the bucket indicating the location for this table'''), Config('bucket_prefix', '', r''' the unique bucket prefix for this table'''), + Config('cache_directory', '', r''' + a directory to store locally cached versions of files in the storage source. By + default, it is named with \c "-cache" appended to the bucket name. A relative + directory name is relative to the home directory'''), ] file_runtime_config = common_runtime_config + [ @@ -996,6 +1004,10 @@ wiredtiger_open_tiered_storage_configuration = [ unique string prefix to identify our objects in the bucket. Multiple instances can share the storage bucket and this identifier is used in naming objects'''), + Config('cache_directory', '', r''' + a directory to store locally cached versions of files in the storage source. By + default, it is named with \c "-cache" appended to the bucket name. A relative + directory name is relative to the home directory'''), Config('name', 'none', r''' Permitted values are \c "none" or custom storage name created with diff --git a/src/third_party/wiredtiger/dist/docs_data.py b/src/third_party/wiredtiger/dist/docs_data.py index 2d0ba1e6068..0929a9dcebd 100644 --- a/src/third_party/wiredtiger/dist/docs_data.py +++ b/src/third_party/wiredtiger/dist/docs_data.py @@ -19,6 +19,11 @@ arch_doc_pages = [ 'WT_BM', 'WT_EXTLIST'], ['src/include/block.h', 'src/include/block_inline.h', 'src/block/']), + ArchDocPage('arch-btree', + ['WT_BTREE', 'WT_PAGE'], + ['src/include/btmem.h', 'src/include/btree.h', + 'src/btree/bt_cursor.c', 'src/btree/bt_delete.c', + 'src/btree/bt_page.c', 'src/btree/bt_read.c']), ArchDocPage('arch-cache', ['WT_CACHE', 'WT_CACHE_POOL', 'WT_COL', 'WT_COL_RLE', 'WT_INSERT', 'WT_PAGE', 'WT_PAGE_MODIFY', 'WT_REF', 'WT_ROW', 'WT_UPDATE'], @@ -44,19 +49,20 @@ arch_doc_pages = [ 'src/include/cell.h', 'src/include/cell_inline.h', 'src/reconcile/rec_col.c', 'src/reconcile/rec_row.c']), ArchDocPage('arch-dhandle', - ['WT_BTREE', 'WT_DHANDLE'], - ['src/include/btree.h', 'src/include/dhandle.h', - 'src/conn/conn_dhandle.c', 'src/session/session_dhandle.c']), + ['WT_DHANDLE'], + ['src/include/dhandle.h', 'src/conn/conn_dhandle.c', + 'src/session/session_dhandle.c']), ArchDocPage('arch-eviction', ['WT_EVICT_ENTRY', 'WT_EVICT_QUEUE'], ['src/include/cache.h', 'src/evict/']), ArchDocPage('arch-fs-os', - ['WT_FILE_SYSTEM'], + ['WT_FILE_HANDLE', 'WT_FILE_SYSTEM'], ['src/include/os.h', 'src/include/os_fhandle_inline.h', 'src/include/os_fs_inline.h', 'src/include/os_fstream_inline.h', - 'src/include/os_windows.h', - 'src/os_posix/', 'src/os_win/']), + 'src/include/os_windows.h', 'src/include/posix.h', + 'src/os_common/', 'src/os_posix/', 'src/os_win/', + ]), ArchDocPage('arch-hs', ['WT_CURSOR_HS'], ['src/history/']), diff --git a/src/third_party/wiredtiger/dist/s_clang-format b/src/third_party/wiredtiger/dist/s_clang-format index a12101f2d53..0bad455c804 100755 --- a/src/third_party/wiredtiger/dist/s_clang-format +++ b/src/third_party/wiredtiger/dist/s_clang-format @@ -7,8 +7,8 @@ trap 'rm -rf $t' 0 1 2 3 13 15 download_clang_format() { if [ `uname` = "Linux" ]; then - curl https://s3.amazonaws.com/boxes.10gen.com/build/build/clang-format-llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.gz -o dist/clang-format.tar.gz - tar --strip=1 -C dist/ -xf dist/clang-format.tar.gz clang-format-llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04/clang-format && rm dist/clang-format.tar.gz + curl https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-llvm-10.0.0-x86_64-linux-gnu-ubuntu-20.04.tar-1.gz -o dist/clang-format.tar.gz + tar --strip=1 -C dist/ -xf dist/clang-format.tar.gz clang-format-llvm-10.0.0-x86_64-linux-gnu-ubuntu-20.04/clang-format && rm dist/clang-format.tar.gz elif [ `uname` = "Darwin" ]; then curl https://s3.amazonaws.com/boxes.10gen.com/build/build/clang-format-llvm-10.0.0-x86_64-apple-darwin.tar.gz -o dist/clang-format.tar.gz tar --strip=1 -C dist/ -xf dist/clang-format.tar.gz clang-format-llvm-10.0.0-x86_64-apple-darwin/clang-format && rm dist/clang-format.tar.gz diff --git a/src/third_party/wiredtiger/dist/s_clang-scan b/src/third_party/wiredtiger/dist/s_clang-scan index 366678e2626..9a5ee3d4587 100644 --- a/src/third_party/wiredtiger/dist/s_clang-scan +++ b/src/third_party/wiredtiger/dist/s_clang-scan @@ -17,9 +17,9 @@ test -z "$scan" || compiler="${SCANCOMPILER:-clang}" if test -z "$scan" && test -x /usr/bin/clang && - test -x /usr/lib/llvm-3.9/bin/scan-build; then + test -x /usr/bin/scan-build; then compiler="/usr/bin/clang" - scan="/usr/lib/llvm-3.9/bin/scan-build" + scan="/usr/bin/scan-build" fi if test -z "$scan"; then @@ -50,9 +50,9 @@ grep 'No bugs found' $t > /dev/null && exit 0 # clang-scan outputs errors/warnings with a leading file path followed by a # colon, followed by the string "error:" or "warning:". cat dist/s_clang-scan.diff | - egrep '^[a-z\./].*error: |^[a-z\./].*warning: ' | sed -e 's/:.*//' > $a + egrep '^[a-z\./].*error: |^[a-z\./].*warning: ' | sed -e 's/:.*:.*://' | sort > $a cat $t | - egrep '^[a-z\./].*error: |^[a-z\./].*warning: ' | sed -e 's/:.*//' > $b + egrep '^[a-z\./].*error: |^[a-z\./].*warning: ' | sed -e 's/:.*:.*://' | sort > $b diff $a $b > /dev/null && exit 0 echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" diff --git a/src/third_party/wiredtiger/dist/s_clang-scan.diff b/src/third_party/wiredtiger/dist/s_clang-scan.diff index 471e383823c..6cc57d04cd5 100644 --- a/src/third_party/wiredtiger/dist/s_clang-scan.diff +++ b/src/third_party/wiredtiger/dist/s_clang-scan.diff @@ -1,23 +1,120 @@ +src/block/block_ckpt_scan.c:273:14: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret' + if ((ret = __wt_read(session, fh, offset, (size_t)WT_BTREE_MIN_ALLOC_SIZE, tmp->mem)) != 0) + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1 warning generated. In file included from src/block/block_write.c:9: -In file included from ./src/include/wt_internal.h:420: +In file included from ./src/include/wt_internal.h:439: ./src/include/intpack_inline.h:193:7: warning: Assigned value is garbage or undefined p = *pp; ^ ~~~ -1 warning generated -In file included from src/btree/col_modify.c:9: -In file included from ./src/include/wt_internal.h:425: -./src/include/mutex_inline.h:158:13: warning: Null pointer passed as an argument to a 'nonnull' parameter - return (pthread_mutex_trylock(&t->lock)); - ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -1 warning generated. -src/conn/conn_capacity.c:288:5: warning: Value stored to 'capacity' is never read +1 warning generated. +src/btree/bt_delete.c:411:5: warning: Null pointer passed to 1st parameter expecting 'nonnull' + __wt_overwrite_and_free(session, ref->ft_info.del); + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +./src/include/misc.h:141:13: note: expanded from macro '__wt_overwrite_and_free' + __wt_explicit_overwrite(p, sizeof(*(p))); \ + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +./src/include/misc.h:135:42: note: expanded from macro '__wt_explicit_overwrite' +#define __wt_explicit_overwrite(p, size) memset(p, WT_DEBUG_BYTE, size) + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/btree/bt_io.c:178:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; + ^ ~ +1 warning generated. +src/btree/bt_read.c:99:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; + ^ ~ +1 warning generated. +src/btree/bt_sync.c:445:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; + ^ ~ +1 warning generated. +src/conn/conn_capacity.c:289:5: warning: Value stored to 'capacity' is never read capacity = steal_capacity = 0; ^ ~~~~~~~~~~~~~~~~~~ 1 warning generated. -In file included from src/reconcile/rec_write.c:9: -In file included from ./src/include/wt_internal.h:425: -./src/include/mutex_inline.h:184:16: warning: Null pointer passed as an argument to a 'nonnull' parameter - if ((ret = pthread_mutex_unlock(&t->lock)) != 0) - ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +src/conn/conn_stat.c:413:11: warning: Although the value stored to 'locked' is used in the enclosing expression, the value is never actually read from 'locked' + cnt = locked = 0; + ^ ~ +1 warning generated. +src/conn/conn_dhandle.c:643:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; + ^ ~ +src/conn/conn_dhandle.c:696:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; + ^ ~ +2 warnings generated. +src/cursor/cur_index.c:369:10: warning: Although the value stored to 'cp' is used in the enclosing expression, the value is never actually read from 'cp' + if ((cp = cindex->cg_cursors) != NULL) + ^ ~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/evict/evict_page.c:105:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; /* [-Werror=maybe-uninitialized] */ + ^ ~ +1 warning generated. +src/evict/evict_stat.c:34:31: warning: Although the value stored to 'size' is used in the enclosing expression, the value is never actually read from 'size' + pages_leaf = seen_count = size = visited_count = 0; + ^ ~~~~~~~~~~~~~~~~~ +1 warning generated. +src/cursor/cur_stat.c:634:10: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret' + if ((ret = __wt_config_gets(session, cfg, "statistics", &cval)) == 0) { + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/history/hs_verify.c:180:14: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret' + if ((ret = __wt_metadata_btree_id_to_uri(session, btree_id, &uri_data)) != 0) { + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/evict/evict_lru.c:2270:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; + ^ ~ +src/evict/evict_lru.c:2336:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; + ^ ~ +2 warnings generated. +src/lsm/lsm_merge.c:341:36: warning: Although the value stored to 'locked' is used in the enclosing expression, the value is never actually read from 'locked' + created_chunk = create_bloom = locked = in_sync = false; + ^ ~~~~~~~~~~~~~~~ +1 warning generated. +src/log/log_slot.c:527:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; + ^ ~ +1 warning generated. +src/schema/schema_alter.c:117:10: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret' + if ((ret = __wt_config_getones(session, value, "source", &cval)) != 0) + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/schema/schema_create.c:371:28: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret' + if (cgname != NULL && (ret = __wt_config_subgets(session, &table->cgconf, cgname, &cval)) != 0) + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +src/schema/schema_create.c:586:14: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret' + if ((ret = __wt_config_getones(session, config, "key_format", &kval)) != 0) + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +2 warnings generated. +src/schema/schema_rename.c:147:10: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret' + if ((ret = __wt_config_getones(session, value, "source", &cval)) != 0) + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/support/hazard.c:322:34: warning: Although the value stored to 'j' is used in the enclosing expression, the value is never actually read from 'j' + for (s = conn->sessions, i = j = max = walk_cnt = 0; i < session_cnt; ++s, ++i) { + ^ ~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/support/mtx_rw.c:181:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; /* -Wconditional-uninitialized */ + ^ ~ +src/support/mtx_rw.c:372:18: warning: Although the value stored to 'time_stop' is used in the enclosing expression, the value is never actually read from 'time_stop' + time_start = time_stop = 0; /* -Wconditional-uninitialized */ + ^ ~ +2 warnings generated. +src/tiered/tiered_handle.c:129:13: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret' + while ((ret = __wt_config_next(&cparser, &ck, &cv)) == 0) { + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/txn/txn_ckpt.c:775:19: warning: Although the value stored to 'logging' is used in the enclosing expression, the value is never actually read from 'logging' + full = idle = logging = tracking = use_timestamp = false; + ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1 warning generated. +src/txn/txn_rollback_to_stable.c:1386:43: warning: Although the value stored to 'has_txn_updates_gt_than_ckpt_snap' is used in the enclosing expression, the value is never actually read from 'has_txn_updates_gt_than_ckpt_snap' + durable_ts_found = prepared_updates = has_txn_updates_gt_than_ckpt_snap = false; + ^ ~~~~~ 1 warning generated. -ar: `u' modifier ignored since `D' is the default (see `U') diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 8b5c02aa8a8..428e7c29375 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -12,11 +12,6 @@ type doxygen > /dev/null 2>&1 || { echo "$0 skipped: doxygen not found" exit 0 } -v=`(echo "1.8.17" && doxygen --version) | sort -V | head -1` -test "$v" = "1.8.17" || { - echo "$0 skipped: unsupported version of doxygen" - exit 0 -} . ../RELEASE_INFO @@ -268,6 +263,14 @@ check_docs_data if [ $e != 0 ]; then exit $e fi + +# Do not build the documentation if Doxygen version is not compatible. +v=$(doxygen --version) +if [ "$v" != "1.8.17" ]; then + echo "$0 skipped: unsupported version of doxygen" + exit 0 +fi + # Build the documentation. build $clean diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 4de70302311..c660bde80c1 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -333,6 +333,7 @@ Pre Preload Prepend Prev +QQQS Qsort RCS RDNOLOCK @@ -1487,6 +1488,7 @@ vsnprintf vtype vunpack vxr +vz waitpid waker wakeup diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index b26301b26bb..6af0af26729 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -252,6 +252,7 @@ conn_stats = [ CacheStat('cache_eviction_walk', 'pages walked for eviction'), CacheStat('cache_eviction_walk_leaf_notfound', 'eviction server waiting for a leaf page'), CacheStat('cache_eviction_walk_passes', 'eviction passes of a file'), + CacheStat('cache_eviction_walk_sleeps', 'eviction walk most recent sleeps for checkpoint handle gathering'), CacheStat('cache_eviction_walks_active', 'files with active eviction walks', 'no_clear,no_scale'), CacheStat('cache_eviction_walks_started', 'files with new eviction walks started'), CacheStat('cache_eviction_worker_created', 'eviction worker thread created'), @@ -485,6 +486,7 @@ conn_stats = [ SessionOpStat('session_table_alter_fail', 'table alter failed calls', 'no_clear,no_scale'), SessionOpStat('session_table_alter_skip', 'table alter unchanged and skipped', 'no_clear,no_scale'), SessionOpStat('session_table_alter_success', 'table alter successful calls', 'no_clear,no_scale'), + SessionOpStat('session_table_alter_trigger_checkpoint', 'table alter triggering checkpoint calls', 'no_clear,no_scale'), SessionOpStat('session_table_compact_fail', 'table compact failed calls', 'no_clear,no_scale'), SessionOpStat('session_table_compact_success', 'table compact successful calls', 'no_clear,no_scale'), SessionOpStat('session_table_create_fail', 'table create failed calls', 'no_clear,no_scale'), diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py index 62789154113..c074877056a 100644 --- a/src/third_party/wiredtiger/dist/test_data.py +++ b/src/third_party/wiredtiger/dist/test_data.py @@ -214,7 +214,10 @@ test_config = [ ] methods = { + 'base_test' : Method(test_config), + 'burst_inserts' : Method(test_config + [ + Config("burst_duration", 90, r''' + How long the insertions will occur for.''')]), 'example_test' : Method(test_config), 'hs_cleanup' : Method(test_config), - 'base_test' : Method(test_config), } diff --git a/src/third_party/wiredtiger/examples/c/ex_file_system.c b/src/third_party/wiredtiger/examples/c/ex_file_system.c index ad43cfe1cc0..9df1f6d73ca 100644 --- a/src/third_party/wiredtiger/examples/c/ex_file_system.c +++ b/src/third_party/wiredtiger/examples/c/ex_file_system.c @@ -228,7 +228,7 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) } allocate_file_system_lock(&demo_fs->lock); - + /*! [WT_FILE_SYSTEM create] */ /* Initialize the in-memory jump table. */ file_system->fs_directory_list = demo_fs_directory_list; file_system->fs_directory_list_free = demo_fs_directory_list_free; @@ -244,6 +244,7 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) wtext, NULL, "WT_CONNECTION.set_file_system: %s", wtext->strerror(wtext, NULL, ret)); goto err; } + /*! [WT_FILE_SYSTEM create] */ return (0); err: @@ -320,7 +321,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, ret = ENOMEM; goto err; } - + /*! [WT_FILE_HANDLE create] */ /* * Setup the function call table for our custom file system. Set the function pointer to NULL * where our implementation doesn't support the functionality. @@ -345,6 +346,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, ++demo_fs->opened_unique_file_count; *file_handlep = file_handle; + /*! [WT_FILE_HANDLE create] */ if (0) { err: diff --git a/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c b/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c index f063a513c71..746ca396298 100644 --- a/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c +++ b/src/third_party/wiredtiger/ext/extractors/csv/csv_extractor.c @@ -147,6 +147,8 @@ csv_customize(WT_EXTRACTOR *extractor, WT_SESSION *session, const char *uri, WT_ orig = (const CSV_EXTRACTOR *)extractor; wt_api = orig->wt_api; + parser = NULL; + if ((ret = wt_api->config_parser_open(wt_api, session, appcfg->str, appcfg->len, &parser)) != 0) return (ret); if ((ret = parser->get(parser, "field", &field)) != 0) { @@ -165,12 +167,6 @@ csv_customize(WT_EXTRACTOR *extractor, WT_SESSION *session, const char *uri, WT_ wt_api->strerror(wt_api, session, ret)); goto err; } - ret = parser->close(parser); - parser = NULL; - if (ret != 0) { - (void)wt_api->err_printf( - wt_api, session, "WT_CONFIG_PARSER.close: %s", wt_api->strerror(wt_api, session, ret)); - } field_num = strtol(field.str, NULL, 10); if (field_num < 0 || field_num > INT_MAX) { @@ -195,8 +191,7 @@ csv_customize(WT_EXTRACTOR *extractor, WT_SESSION *session, const char *uri, WT_ return (0); err: - if (parser != NULL) - (void)parser->close(parser); + (void)parser->close(parser); return (ret); } diff --git a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c index 11384645222..8bcb6911d06 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c +++ b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c @@ -103,9 +103,10 @@ typedef struct { /* This is WiredTiger's file system, it is used in implementing the local file system. */ WT_FILE_SYSTEM *wt_fs; - char *auth_token; /* Identifier for key management system */ - char *bucket_dir; /* Directory that stands in for cloud storage bucket */ - char *cache_dir; /* Directory for pre-flushed objects and cached objects */ + char *auth_token; /* Identifier for key management system */ + char *bucket_dir; /* Directory that stands in for cloud storage bucket */ + char *cache_dir; /* Directory for cached objects */ + const char *home_dir; /* Owned by the connection */ } LOCAL_FILE_SYSTEM; typedef struct local_file_handle { @@ -128,7 +129,7 @@ static int local_delay(LOCAL_STORAGE *); static int local_err(LOCAL_STORAGE *, WT_SESSION *, int, const char *, ...); static int local_file_copy( LOCAL_STORAGE *, WT_SESSION *, const char *, const char *, WT_FS_OPEN_FILE_TYPE); -static int local_get_directory(const char *, ssize_t len, char **); +static int local_get_directory(const char *, const char *, ssize_t len, bool, char **); static int local_path(WT_FILE_SYSTEM *, const char *, const char *, char **); static int local_stat( WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, bool, struct stat *); @@ -175,18 +176,13 @@ static int local_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); static int local_file_sync(WT_FILE_HANDLE *, WT_SESSION *); static int local_file_write(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, const void *); -/* - * Report an error for a file operation. Note that local_err returns its third argument, and this - * macro will too. - */ -#define WT_FS2LOCAL(fs) (((LOCAL_FILE_SYSTEM *)(fs))->local_storage) - -#define WT_VERBOSE_LS(local, ...) \ +#define FS2LOCAL(fs) (((LOCAL_FILE_SYSTEM *)(fs))->local_storage) +#define SHOW_STRING(s) (((s) == NULL) ? "<null>" : (s)) +#define VERBOSE_LS(local, ...) \ do { \ if ((local)->verbose > 0) \ fprintf(stderr, __VA_ARGS__); \ } while (0); -#define WT_SHOW_STRING(s) (((s) == NULL) ? "<null>" : (s)) /* * local_configure @@ -247,7 +243,7 @@ local_delay(LOCAL_STORAGE *local) ret = 0; if (local->force_delay != 0 && (local->object_reads + local->object_writes) % local->force_delay == 0) { - WT_VERBOSE_LS(local, + VERBOSE_LS(local, "Artificial delay %" PRIu32 " milliseconds after %" PRIu64 " object reads, %" PRIu64 " object writes\n", local->delay_ms, local->object_reads, local->object_writes); @@ -263,7 +259,7 @@ local_delay(LOCAL_STORAGE *local) } if (local->force_error != 0 && (local->object_reads + local->object_writes) % local->force_error == 0) { - WT_VERBOSE_LS(local, + VERBOSE_LS(local, "Artificial error returned after %" PRIu64 " object reads, %" PRIu64 " object writes\n", local->object_reads, local->object_writes); ret = ENETUNREACH; @@ -285,7 +281,7 @@ local_err(LOCAL_STORAGE *local, WT_SESSION *session, int ret, const char *format va_start(ap, format); wt_api = local->wt_api; - if (vsnprintf(buf, sizeof(buf), format, ap) > (int)sizeof(buf)) + if (vsnprintf(buf, sizeof(buf), format, ap) >= (int)sizeof(buf)) wt_api->err_printf(wt_api, session, "local_storage: error overflow"); wt_api->err_printf( wt_api, session, "local_storage: %s: %s", wt_api->strerror(wt_api, session, ret), buf); @@ -299,18 +295,35 @@ local_err(LOCAL_STORAGE *local, WT_SESSION *session, int ret, const char *format * Return a copy of a directory name after verifying that it is a directory. */ static int -local_get_directory(const char *s, ssize_t len, char **copy) +local_get_directory(const char *home, const char *s, ssize_t len, bool create, char **copy) { struct stat sb; + size_t buflen; int ret; char *dirname; + *copy = NULL; + if (len == -1) len = (ssize_t)strlen(s); - dirname = strndup(s, (size_t)len + 1); /* Room for null */ + + /* For relative pathnames, the path is considered to be relative to the home directory. */ + if (*s == '/') + dirname = strndup(s, (size_t)len + 1); /* Room for null */ + else { + buflen = (size_t)len + strlen(home) + 2; /* Room for slash, null */ + if ((dirname = malloc(buflen)) != NULL) + if (snprintf(dirname, buflen, "%s/%.*s", home, (int)len, s) >= (int)buflen) + return (EINVAL); + } if (dirname == NULL) return (ENOMEM); + ret = stat(dirname, &sb); + if (ret != 0 && errno == ENOENT && create) { + (void)mkdir(dirname, 0777); + ret = stat(dirname, &sb); + } if (ret != 0) ret = errno; else if ((sb.st_mode & S_IFMT) != S_IFDIR) @@ -365,8 +378,9 @@ local_path(WT_FILE_SYSTEM *file_system, const char *dir, const char *name, char } len = strlen(dir) + strlen(name) + 2; if ((p = malloc(len)) == NULL) - return (local_err(WT_FS2LOCAL(file_system), NULL, ENOMEM, "local_path")); - snprintf(p, len, "%s/%s", dir, name); + return (local_err(FS2LOCAL(file_system), NULL, ENOMEM, "local_path")); + if (snprintf(p, len, "%s/%s", dir, name) >= (int)len) + return (local_err(FS2LOCAL(file_system), NULL, EINVAL, "overflow sprintf")); *pathp = p; return (ret); } @@ -405,7 +419,7 @@ local_stat(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, c * If the file must exist, report the error no matter what. */ if (must_exist || errno != ENOENT) - ret = local_err(WT_FS2LOCAL(file_system), session, errno, "%s: %s stat", path, caller); + ret = local_err(FS2LOCAL(file_system), session, errno, "%s: %s stat", path, caller); else ret = errno; } @@ -485,10 +499,18 @@ local_customize_file_system(WT_STORAGE_SOURCE *storage_source, WT_SESSION *sessi ret = local_err(local, session, ENOMEM, "local_file_system.auth_token"); goto err; } + + /* + * The home directory owned by the connection will not change, and will be valid memory, for as + * long as the connection is open. That is longer than this file system will be open, so we can + * use the string without copying. + */ + fs->home_dir = session->connection->get_home(session->connection); + /* * Get the bucket directory and the cache directory. */ - if ((ret = local_get_directory(bucket_name, -1, &fs->bucket_dir)) != 0) { + if ((ret = local_get_directory(fs->home_dir, bucket_name, -1, false, &fs->bucket_dir)) != 0) { ret = local_err(local, session, ret, "%s: bucket directory", bucket_name); goto err; } @@ -502,12 +524,15 @@ local_customize_file_system(WT_STORAGE_SOURCE *storage_source, WT_SESSION *sessi p++; else p = bucket_name; - snprintf(buf, sizeof(buf), "cache-%s", p); + if (snprintf(buf, sizeof(buf), "cache-%s", p) >= (int)sizeof(buf)) { + ret = local_err(local, session, EINVAL, "overflow snprintf"); + goto err; + } cachedir.str = buf; cachedir.len = strlen(buf); - (void)mkdir(buf, 0777); } - if ((ret = local_get_directory(cachedir.str, (ssize_t)cachedir.len, &fs->cache_dir)) != 0) { + if ((ret = local_get_directory( + fs->home_dir, cachedir.str, (ssize_t)cachedir.len, true, &fs->cache_dir)) != 0) { ret = local_err(local, session, ret, "%*s: cache directory", (int)cachedir.len, cachedir.str); goto err; @@ -545,7 +570,7 @@ local_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, LOCAL_STORAGE *local; int ret; - local = WT_FS2LOCAL(file_system); + local = FS2LOCAL(file_system); local->op_count++; *existp = false; @@ -569,10 +594,17 @@ local_file_copy(LOCAL_STORAGE *local, WT_SESSION *session, const char *src_path, WT_FILE_SYSTEM *wt_fs; wt_off_t copy_size, file_size, left; ssize_t pos; + size_t pathlen; int ret, t_ret; - char buffer[1024 * 64]; + char buffer[1024 * 64], *tmp_path; dest = src = NULL; + pathlen = strlen(dest_path) + 10; + if ((tmp_path = malloc(pathlen)) != NULL) + if (snprintf(tmp_path, pathlen, "%s.TMP", dest_path) >= (int)pathlen) { + ret = local_err(local, session, EINVAL, "overflow snprintf"); + goto err; + } if ((ret = local->wt_api->file_system_get(local->wt_api, session, &wt_fs)) != 0) { ret = @@ -585,9 +617,9 @@ local_file_copy(LOCAL_STORAGE *local, WT_SESSION *session, const char *src_path, goto err; } - if ((ret = wt_fs->fs_open_file(wt_fs, session, dest_path, type, WT_FS_OPEN_CREATE, &dest)) != + if ((ret = wt_fs->fs_open_file(wt_fs, session, tmp_path, type, WT_FS_OPEN_CREATE, &dest)) != 0) { - ret = local_err(local, session, ret, "%s: cannot create", dest_path); + ret = local_err(local, session, ret, "%s: cannot create", tmp_path); goto err; } if ((ret = wt_fs->fs_size(wt_fs, session, src_path, &file_size)) != 0) { @@ -601,10 +633,14 @@ local_file_copy(LOCAL_STORAGE *local, WT_SESSION *session, const char *src_path, goto err; } if ((ret = dest->fh_write(dest, session, pos, (size_t)copy_size, buffer)) != 0) { - ret = local_err(local, session, ret, "%s: cannot write", dest_path); + ret = local_err(local, session, ret, "%s: cannot write", tmp_path); goto err; } } + if ((ret = rename(tmp_path, dest_path)) != 0) { + ret = local_err(local, session, errno, "%s: cannot rename from %s", dest_path, tmp_path); + goto err; + } err: if (src != NULL && (t_ret = src->close(src, session)) != 0) if (ret == 0) @@ -612,6 +648,9 @@ err: if (dest != NULL && (t_ret = dest->close(dest, session)) != 0) if (ret == 0) ret = t_ret; + if (ret != 0) + (void)unlink(tmp_path); + free(tmp_path); return (ret); } @@ -697,7 +736,7 @@ static int local_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp) { - WT_FS2LOCAL(file_system)->op_count++; + FS2LOCAL(file_system)->op_count++; return ( local_directory_list_internal(file_system, session, directory, prefix, 0, dirlistp, countp)); } @@ -710,7 +749,7 @@ static int local_directory_list_single(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp) { - WT_FS2LOCAL(file_system)->op_count++; + FS2LOCAL(file_system)->op_count++; return ( local_directory_list_internal(file_system, session, directory, prefix, 1, dirlistp, countp)); } @@ -725,7 +764,7 @@ local_directory_list_free( { (void)session; - WT_FS2LOCAL(file_system)->op_count++; + FS2LOCAL(file_system)->op_count++; if (dirlist != NULL) { while (count > 0) free(dirlist[--count]); @@ -853,7 +892,7 @@ local_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) (void)session; /* unused */ local_fs = (LOCAL_FILE_SYSTEM *)file_system; - WT_FS2LOCAL(file_system)->op_count++; + FS2LOCAL(file_system)->op_count++; free(local_fs->auth_token); free(local_fs->bucket_dir); free(local_fs->cache_dir); @@ -983,8 +1022,8 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, *file_handlep = file_handle; - WT_VERBOSE_LS(local, "File opened: %s final path=%s\n", WT_SHOW_STRING(name), - WT_SHOW_STRING(local_fh->fh->name)); + VERBOSE_LS( + local, "File opened: %s final path=%s\n", SHOW_STRING(name), SHOW_STRING(local_fh->fh->name)); err: free(bucket_path); @@ -1007,8 +1046,8 @@ local_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *from, (void)to; /* unused */ (void)flags; /* unused */ - return (local_err( - WT_FS2LOCAL(file_system), session, ENOTSUP, "%s: rename of file not supported", from)); + return ( + local_err(FS2LOCAL(file_system), session, ENOTSUP, "%s: rename of file not supported", from)); } /* @@ -1020,8 +1059,8 @@ local_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, { (void)flags; /* unused */ - return (local_err( - WT_FS2LOCAL(file_system), session, ENOTSUP, "%s: remove of file not supported", name)); + return ( + local_err(FS2LOCAL(file_system), session, ENOTSUP, "%s: remove of file not supported", name)); } /* @@ -1035,7 +1074,7 @@ local_size(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, w LOCAL_STORAGE *local; int ret; - local = WT_FS2LOCAL(file_system); + local = FS2LOCAL(file_system); local->op_count++; *sizep = 0; diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 1d385104d0a..ffc75973557 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "9893156289e444466848ae5d2b5ada7a4e0d6c0d" + "commit": "a9bd69a7ca5dd14ae36ab76574ac7eb1c0207758" } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index ff4c436a6e1..8506083e006 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -468,8 +468,12 @@ restart_read_page: static int __cursor_key_order_check_col(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool next) { + WT_BTREE *btree; + WT_DECL_RET; int cmp; + WT_UNUSED(ret); + btree = S2BT(session); cmp = 0; /* -Werror=maybe-uninitialized */ if (cbt->lastrecno != WT_RECNO_OOB) { @@ -484,11 +488,14 @@ __cursor_key_order_check_col(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo return (0); } - WT_RET(__wt_msg(session, "dumping the cursor page")); - WT_RET(__wt_debug_cursor_page(&cbt->iface, NULL)); - WT_RET_PANIC(session, EINVAL, + WT_RET(__wt_msg(session, "dumping the tree")); + WT_WITH_BTREE(session, btree, ret = __wt_debug_tree_all(session, NULL, NULL, NULL)); + WT_ERR_PANIC(session, EINVAL, "WT_CURSOR.%s out-of-order returns: returned key %" PRIu64 " then key %" PRIu64, next ? "next" : "prev", cbt->lastrecno, cbt->recno); + +err: + return (ret); } /* @@ -524,8 +531,8 @@ __cursor_key_order_check_row(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo __wt_buf_set_printable_format( session, cbt->lastkey->data, cbt->lastkey->size, btree->key_format, false, a), __wt_buf_set_printable_format(session, key->data, key->size, btree->key_format, false, b))); - WT_ERR(__wt_msg(session, "dumping the cursor page")); - WT_ERR(__wt_debug_cursor_page(&cbt->iface, NULL)); + WT_ERR(__wt_msg(session, "dumping the tree")); + WT_WITH_BTREE(session, btree, ret = __wt_debug_tree_all(session, NULL, NULL, NULL)); WT_ERR_PANIC(session, EINVAL, "found key out-of-order returns"); err: diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index c952272bb8b..b3d879fae8f 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -684,6 +684,14 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) } /* + * If a valid key has been found and we are doing a prefix search near, we want to return the + * key only if it is a prefix match. + */ + if (valid && F_ISSET(cursor, WT_CURSTD_PREFIX_SEARCH) && + __wt_prefix_match(&state.key, cbt->tmp) != 0) + valid = false; + + /* * If we find a valid key, return it. * * Else, creating a record past the end of the tree in a fixed-length column-store implicitly diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index e2205b78047..2e594da3388 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -14,6 +14,7 @@ */ typedef struct __wt_dbg WT_DBG; struct __wt_dbg { + WT_CURSOR *hs_cursor; WT_SESSION_IMPL *session; /* Enclosing session */ WT_ITEM *key; @@ -46,12 +47,12 @@ static int __debug_modify(WT_DBG *, const uint8_t *); static int __debug_page(WT_DBG *, WT_REF *, uint32_t); static int __debug_page_col_fix(WT_DBG *, WT_REF *); static int __debug_page_col_int(WT_DBG *, WT_PAGE *, uint32_t); -static int __debug_page_col_var(WT_DBG *, WT_REF *, WT_CURSOR *); +static int __debug_page_col_var(WT_DBG *, WT_REF *); static int __debug_page_metadata(WT_DBG *, WT_REF *); static int __debug_page_row_int(WT_DBG *, WT_PAGE *, uint32_t); -static int __debug_page_row_leaf(WT_DBG *, WT_PAGE *, WT_CURSOR *); +static int __debug_page_row_leaf(WT_DBG *, WT_PAGE *); static int __debug_ref(WT_DBG *, WT_REF *); -static int __debug_row_skip(WT_DBG *, WT_INSERT_HEAD *, WT_CURSOR *); +static int __debug_row_skip(WT_DBG *, WT_INSERT_HEAD *); static int __debug_tree(WT_SESSION_IMPL *, WT_REF *, const char *, uint32_t); static int __debug_update(WT_DBG *, WT_UPDATE *, bool); static int __debug_wrapup(WT_DBG *); @@ -238,16 +239,31 @@ static int __debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) { WT_BTREE *btree; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; memset(ds, 0, sizeof(WT_DBG)); ds->session = session; + ds->hs_cursor = NULL; + conn = S2C(session); WT_ERR(__wt_scr_alloc(session, 512, &ds->t1)); WT_ERR(__wt_scr_alloc(session, 512, &ds->t2)); /* + * Set up history store support, opening a history store cursor on demand. Return error if that + * doesn't work, except while running in-memory configuration. + */ + if (!F_ISSET(conn, WT_CONN_IN_MEMORY) && !WT_IS_HS(session->dhandle)) + WT_ERR(__wt_curhs_open(session, NULL, &ds->hs_cursor)); + + if (ds->hs_cursor != NULL) { + F_SET(ds->hs_cursor, WT_CURSTD_HS_READ_COMMITTED); + WT_ERR(__wt_scr_alloc(session, 0, &ds->hs_key)); + WT_ERR(__wt_scr_alloc(session, 0, &ds->hs_value)); + } + /* * If we weren't given a file, we use the default event handler, and we'll have to buffer * messages. */ @@ -291,6 +307,9 @@ __debug_wrapup(WT_DBG *ds) __wt_scr_free(session, &ds->t1); __wt_scr_free(session, &ds->t2); + if (ds->hs_cursor != NULL) + WT_TRET(ds->hs_cursor->close(ds->hs_cursor)); + /* * Discard the buffer -- it shouldn't have anything in it, but might as well be cautious. */ @@ -462,7 +481,7 @@ __debug_hs_cursor(WT_DBG *ds, WT_CURSOR *hs_cursor) * Dump any HS records associated with the key. */ static int -__debug_hs_key(WT_DBG *ds, WT_CURSOR *hs_cursor) +__debug_hs_key(WT_DBG *ds) { WT_BTREE *btree; WT_DECL_RET; @@ -477,11 +496,11 @@ __debug_hs_key(WT_DBG *ds, WT_CURSOR *hs_cursor) * Open a history store cursor positioned at the end of the data store key (the newest record) * and iterate backwards until we reach a different key or btree. */ - hs_cursor->set_key(hs_cursor, 4, hs_btree_id, ds->key, WT_TS_MAX, WT_TXN_MAX); - ret = __wt_curhs_search_near_before(session, hs_cursor); + ds->hs_cursor->set_key(ds->hs_cursor, 4, hs_btree_id, ds->key, WT_TS_MAX, WT_TXN_MAX); + ret = __wt_curhs_search_near_before(session, ds->hs_cursor); - for (; ret == 0; ret = hs_cursor->prev(hs_cursor)) - WT_RET(__debug_hs_cursor(ds, hs_cursor)); + for (; ret == 0; ret = ds->hs_cursor->prev(ds->hs_cursor)) + WT_RET(__debug_hs_cursor(ds, ds->hs_cursor)); return (ret == WT_NOTFOUND ? 0 : ret); } @@ -919,10 +938,11 @@ __wt_debug_page(void *session_arg, WT_BTREE *btree, WT_REF *ref, const char *ofi ds = &_ds; WT_WITH_BTREE(session, btree, ret = __debug_config(session, ds, ofile)); - WT_RET(ret); + WT_ERR(ret); WT_WITH_BTREE(session, btree, ret = __debug_page(ds, ref, WT_DEBUG_TREE_LEAF)); +err: WT_TRET(__debug_wrapup(ds)); return (ret); } @@ -979,7 +999,7 @@ __debug_tree(WT_SESSION_IMPL *session, WT_REF *ref, const char *ofile, uint32_t WT_DECL_RET; ds = &_ds; - WT_RET(__debug_config(session, ds, ofile)); + WT_ERR(__debug_config(session, ds, ofile)); /* A NULL page starts at the top of the tree -- it's a convenience. */ if (ref == NULL) @@ -987,6 +1007,7 @@ __debug_tree(WT_SESSION_IMPL *session, WT_REF *ref, const char *ofile, uint32_t ret = __debug_page(ds, ref, flags); +err: WT_TRET(__debug_wrapup(ds)); return (ret); } @@ -998,26 +1019,12 @@ __debug_tree(WT_SESSION_IMPL *session, WT_REF *ref, const char *ofile, uint32_t static int __debug_page(WT_DBG *ds, WT_REF *ref, uint32_t flags) { - WT_CURSOR *hs_cursor; WT_DECL_RET; WT_SESSION_IMPL *session; - hs_cursor = NULL; session = ds->session; WT_RET(__wt_scr_alloc(session, 100, &ds->key)); - /* - * Set up history store support, opening a history store cursor on demand. Ignore errors if that - * doesn't work, we may be running in-memory. - */ - if (!WT_IS_HS(session->dhandle)) { - WT_ERR(__wt_curhs_open(session, NULL, &hs_cursor)); - F_SET(hs_cursor, WT_CURSTD_HS_READ_COMMITTED); - WT_ERR(__wt_scr_alloc(session, 0, &ds->hs_key)); - WT_ERR(__wt_scr_alloc(session, 0, &ds->hs_value)); - F_SET(hs_cursor, WT_CURSTD_HS_READ_COMMITTED); - } - /* Dump the page metadata. */ WT_WITH_PAGE_INDEX(session, ret = __debug_page_metadata(ds, ref)); WT_ERR(ret); @@ -1034,7 +1041,7 @@ __debug_page(WT_DBG *ds, WT_REF *ref, uint32_t flags) break; case WT_PAGE_COL_VAR: if (LF_ISSET(WT_DEBUG_TREE_LEAF)) - WT_ERR(__debug_page_col_var(ds, ref, hs_cursor)); + WT_ERR(__debug_page_col_var(ds, ref)); break; case WT_PAGE_ROW_INT: WT_WITH_PAGE_INDEX(session, ret = __debug_page_row_int(ds, ref->page, flags)); @@ -1042,15 +1049,13 @@ __debug_page(WT_DBG *ds, WT_REF *ref, uint32_t flags) break; case WT_PAGE_ROW_LEAF: if (LF_ISSET(WT_DEBUG_TREE_LEAF)) - WT_ERR(__debug_page_row_leaf(ds, ref->page, hs_cursor)); + WT_ERR(__debug_page_row_leaf(ds, ref->page)); break; default: WT_ERR(__wt_illegal_value(session, ref->page->type)); } err: - if (hs_cursor != NULL) - WT_TRET(hs_cursor->close(hs_cursor)); return (ret); } @@ -1233,7 +1238,6 @@ __debug_page_col_int(WT_DBG *ds, WT_PAGE *page, uint32_t flags) } WT_INTL_FOREACH_END; } - return (0); } @@ -1242,7 +1246,7 @@ __debug_page_col_int(WT_DBG *ds, WT_PAGE *page, uint32_t flags) * Dump an in-memory WT_PAGE_COL_VAR page. */ static int -__debug_page_col_var(WT_DBG *ds, WT_REF *ref, WT_CURSOR *hs_cursor) +__debug_page_col_var(WT_DBG *ds, WT_REF *ref) { WT_CELL *cell; WT_CELL_UNPACK_KV *unpack, _unpack; @@ -1267,21 +1271,21 @@ __debug_page_col_var(WT_DBG *ds, WT_REF *ref, WT_CURSOR *hs_cursor) WT_RET(__wt_snprintf(tag, sizeof(tag), "%" PRIu64 " %" PRIu64, recno, rle)); WT_RET(__debug_cell_kv(ds, page, WT_PAGE_COL_VAR, tag, unpack)); - if (!WT_IS_HS(session->dhandle)) { + if (!WT_IS_HS(session->dhandle) && ds->hs_cursor != NULL) { p = ds->key->mem; WT_RET(__wt_vpack_uint(&p, 0, recno)); ds->key->size = WT_PTRDIFF(p, ds->key->mem); - WT_RET(__debug_hs_key(ds, hs_cursor)); + WT_RET(__debug_hs_key(ds)); } if ((update = WT_COL_UPDATE(page, cip)) != NULL) - WT_RET(__debug_col_skip(ds, update, "update", false, hs_cursor)); + WT_RET(__debug_col_skip(ds, update, "update", false, ds->hs_cursor)); recno += rle; } if (WT_COL_APPEND(page) != NULL) { WT_RET(ds->f(ds, "%s", sep)); - WT_RET(__debug_col_skip(ds, WT_COL_APPEND(page), "append", false, hs_cursor)); + WT_RET(__debug_col_skip(ds, WT_COL_APPEND(page), "append", false, ds->hs_cursor)); } return (0); @@ -1325,7 +1329,7 @@ __debug_page_row_int(WT_DBG *ds, WT_PAGE *page, uint32_t flags) * Dump an in-memory WT_PAGE_ROW_LEAF page. */ static int -__debug_page_row_leaf(WT_DBG *ds, WT_PAGE *page, WT_CURSOR *hs_cursor) +__debug_page_row_leaf(WT_DBG *ds, WT_PAGE *page) { WT_CELL_UNPACK_KV *unpack, _unpack; WT_INSERT_HEAD *insert; @@ -1341,7 +1345,7 @@ __debug_page_row_leaf(WT_DBG *ds, WT_PAGE *page, WT_CURSOR *hs_cursor) * Dump any K/V pairs inserted into the page before the first from-disk key on the page. */ if ((insert = WT_ROW_INSERT_SMALLEST(page)) != NULL) - WT_RET(__debug_row_skip(ds, insert, hs_cursor)); + WT_RET(__debug_row_skip(ds, insert)); /* Dump the page's K/V pairs. */ WT_ROW_FOREACH (page, rip, i) { @@ -1354,11 +1358,11 @@ __debug_page_row_leaf(WT_DBG *ds, WT_PAGE *page, WT_CURSOR *hs_cursor) if ((upd = WT_ROW_UPDATE(page, rip)) != NULL) WT_RET(__debug_update(ds, upd, false)); - if (!WT_IS_HS(session->dhandle) && hs_cursor != NULL) - WT_RET(__debug_hs_key(ds, hs_cursor)); + if (!WT_IS_HS(session->dhandle) && ds->hs_cursor != NULL) + WT_RET(__debug_hs_key(ds)); if ((insert = WT_ROW_INSERT(page, rip)) != NULL) - WT_RET(__debug_row_skip(ds, insert, hs_cursor)); + WT_RET(__debug_row_skip(ds, insert)); } return (0); } @@ -1385,7 +1389,7 @@ __debug_col_skip( p = ds->key->mem; WT_RET(__wt_vpack_uint(&p, 0, WT_INSERT_RECNO(ins))); ds->key->size = WT_PTRDIFF(p, ds->key->mem); - WT_RET(__debug_hs_key(ds, hs_cursor)); + WT_RET(__debug_hs_key(ds)); } } return (0); @@ -1396,7 +1400,7 @@ __debug_col_skip( * Dump an insert list. */ static int -__debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head, WT_CURSOR *hs_cursor) +__debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head) { WT_INSERT *ins; WT_SESSION_IMPL *session; @@ -1407,9 +1411,9 @@ __debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head, WT_CURSOR *hs_cursor) WT_RET(__debug_item_key(ds, "insert", WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins))); WT_RET(__debug_update(ds, ins->upd, false)); - if (!WT_IS_HS(session->dhandle) && hs_cursor != NULL) { + if (!WT_IS_HS(session->dhandle) && ds->hs_cursor != NULL) { WT_RET(__wt_buf_set(session, ds->key, WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins))); - WT_RET(__debug_hs_key(ds, hs_cursor)); + WT_RET(__debug_hs_key(ds)); } } return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index acb89293002..f35346db415 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -305,12 +305,14 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) /* * Give the page a modify structure. - * - * Mark tree dirty, unless the handle is read-only. (We'd like to free the deleted pages, but if - * the handle is read-only, we're not able to do so.) */ WT_RET(__wt_page_modify_init(session, page)); - if (!F_ISSET(btree, WT_BTREE_READONLY)) + + /* + * We would like to free the deleted pages, but if the tree is newly created, there is nothing + * that needs to be freed. Furthermore, if the handle is read-only, we are not able to do so. + */ + if (!btree->original) __wt_page_modify_set(session, page); /* diff --git a/src/third_party/wiredtiger/src/btree/bt_ret.c b/src/third_party/wiredtiger/src/btree/bt_ret.c index b9d14f0a889..3709c112c2b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ret.c +++ b/src/third_party/wiredtiger/src/btree/bt_ret.c @@ -109,25 +109,36 @@ __wt_read_row_time_window(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, * Read the time window from the cell. */ void -__wt_read_cell_time_window(WT_CURSOR_BTREE *cbt, WT_REF *ref, WT_TIME_WINDOW *tw) +__wt_read_cell_time_window(WT_CURSOR_BTREE *cbt, WT_TIME_WINDOW *tw, bool *tw_foundp) { WT_PAGE *page; WT_SESSION_IMPL *session; + *tw_foundp = false; + session = CUR2S(cbt); - page = ref->page; + page = cbt->ref->page; - WT_ASSERT(session, tw != NULL); + if (cbt->slot == UINT32_MAX) + return; /* Take the value from the original page cell. */ - if (page->type == WT_PAGE_ROW_LEAF) { + switch (page->type) { + case WT_PAGE_ROW_LEAF: + if (page->pg_row == NULL) + return; __wt_read_row_time_window(session, page, &page->pg_row[cbt->slot], tw); - } else if (page->type == WT_PAGE_COL_VAR) { + break; + case WT_PAGE_COL_VAR: + if (page->pg_var == NULL) + return; __read_col_time_window(session, page, WT_COL_PTR(page, &page->pg_var[cbt->slot]), tw); - } else { - /* WT_PAGE_COL_FIX: return the default time window. */ - WT_TIME_WINDOW_INIT(tw); + break; + default: /* WT_PAGE_COL_FIX */ + /* no time windows yet */ + return; } + *tw_foundp = true; } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index f7ca3b9f080..2c2b255242c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -1421,7 +1421,8 @@ __split_multi_inmem(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *multi, WT continue; if (supd->ins == NULL) { - slot = WT_ROW_SLOT(orig, supd->ripcip); + /* Note: supd->ins is never null for column-store. */ + slot = WT_ROW_SLOT(orig, supd->rip); upd = orig->modify->mod_row_update[slot]; } else upd = supd->ins->upd; @@ -1474,7 +1475,7 @@ __split_multi_inmem(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *multi, WT case WT_PAGE_ROW_LEAF: /* Build a key. */ if (supd->ins == NULL) - WT_ERR(__wt_row_leaf_key(session, orig, supd->ripcip, key, false)); + WT_ERR(__wt_row_leaf_key(session, orig, supd->rip, key, false)); else { key->data = WT_INSERT_KEY(supd->ins); key->size = WT_INSERT_KEY_SIZE(supd->ins); @@ -1551,7 +1552,8 @@ __split_multi_inmem_final(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *mul continue; if (supd->ins == NULL) { - slot = WT_ROW_SLOT(orig, supd->ripcip); + /* Note: supd->ins is never null for column-store. */ + slot = WT_ROW_SLOT(orig, supd->rip); orig->modify->mod_row_update[slot] = NULL; } else supd->ins->upd = NULL; @@ -1586,7 +1588,8 @@ __split_multi_inmem_fail(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *mult continue; if (supd->ins == NULL) { - slot = WT_ROW_SLOT(orig, supd->ripcip); + /* Note: supd->ins is never null for column-store. */ + slot = WT_ROW_SLOT(orig, supd->rip); upd = orig->modify->mod_row_update[slot]; } else upd = supd->ins->upd; diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index d25699df070..f316863e74a 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -20,22 +20,27 @@ static int __verify_dsk_row_int(WT_SESSION_IMPL *, const char *, const WT_PAGE_H static int __verify_dsk_row_leaf( WT_SESSION_IMPL *, const char *, const WT_PAGE_HEADER *, WT_ADDR *); -#define WT_ERR_VRFY(session, ...) \ - do { \ - if (!(F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE))) \ - __wt_errx(session, __VA_ARGS__); \ - goto err; \ +#define WT_ERR_VRFY(session, ...) \ + do { \ + if (!(F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE))) { \ + __wt_errx(session, __VA_ARGS__); \ + /* Easy way to set a breakpoint when tracking corruption */ \ + WT_IGNORE_RET(__wt_session_breakpoint((WT_SESSION *)session)); \ + } \ + goto err; \ } while (0) -#define WT_RET_VRFY_RETVAL(session, ret, ...) \ - do { \ - if (!(F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE))) { \ - if ((ret) == 0) \ - __wt_errx(session, __VA_ARGS__); \ - else \ - __wt_err(session, ret, __VA_ARGS__); \ - } \ - return ((ret) == 0 ? WT_ERROR : ret); \ +#define WT_RET_VRFY_RETVAL(session, ret, ...) \ + do { \ + if (!(F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE))) { \ + if ((ret) == 0) \ + __wt_errx(session, __VA_ARGS__); \ + else \ + __wt_err(session, ret, __VA_ARGS__); \ + /* Easy way to set a breakpoint when tracking corruption */ \ + WT_IGNORE_RET(__wt_session_breakpoint((WT_SESSION *)session)); \ + } \ + return ((ret) == 0 ? WT_ERROR : ret); \ } while (0) #define WT_RET_VRFY(session, ...) WT_RET_VRFY_RETVAL(session, 0, __VA_ARGS__) diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index cb0a104dc6b..9b95fe6762d 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -265,6 +265,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create_lsm_subconfigs[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_create_tiered_storage_subconfigs[] = { {"auth_token", "string", NULL, NULL, NULL, 0}, {"bucket", "string", NULL, NULL, NULL, 0}, {"bucket_prefix", "string", NULL, NULL, NULL, 0}, + {"cache_directory", "string", NULL, NULL, NULL, 0}, {"local_retention", "int", NULL, "min=0,max=10000", NULL, 0}, {"name", "string", NULL, NULL, NULL, 0}, {"object_target_size", "int", NULL, "min=100K,max=10TB", NULL, 0}, @@ -314,7 +315,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"tiered_object", "boolean", NULL, NULL, NULL, 0}, {"tiered_storage", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_storage_subconfigs, - 6}, + 7}, {"type", "string", NULL, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, @@ -469,7 +470,7 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"tiered_object", "boolean", NULL, NULL, NULL, 0}, {"tiered_storage", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_storage_subconfigs, - 6}, + 7}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, {"write_timestamp_usage", "string", NULL, @@ -521,7 +522,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"tiered_object", "boolean", NULL, NULL, NULL, 0}, {"tiered_storage", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_storage_subconfigs, - 6}, + 7}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, {"version", "string", NULL, NULL, NULL, 0}, @@ -589,7 +590,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"tiered_object", "boolean", NULL, NULL, NULL, 0}, {"tiered_storage", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_storage_subconfigs, - 6}, + 7}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, {"write_timestamp_usage", "string", NULL, @@ -642,7 +643,7 @@ static const WT_CONFIG_CHECK confchk_object_meta[] = { {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"tiered_object", "boolean", NULL, NULL, NULL, 0}, {"tiered_storage", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_storage_subconfigs, - 6}, + 7}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, {"version", "string", NULL, NULL, NULL, 0}, @@ -674,6 +675,7 @@ static const WT_CONFIG_CHECK confchk_tier_meta[] = { {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"bucket", "string", NULL, NULL, NULL, 0}, {"bucket_prefix", "string", NULL, NULL, NULL, 0}, + {"cache_directory", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_backup_info", "string", NULL, NULL, NULL, 0}, {"checkpoint_lsn", "string", NULL, NULL, NULL, 0}, @@ -710,7 +712,7 @@ static const WT_CONFIG_CHECK confchk_tier_meta[] = { {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"tiered_object", "boolean", NULL, NULL, NULL, 0}, {"tiered_storage", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_storage_subconfigs, - 6}, + 7}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, {"version", "string", NULL, NULL, NULL, 0}, @@ -763,7 +765,7 @@ static const WT_CONFIG_CHECK confchk_tiered_meta[] = { {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"tiered_object", "boolean", NULL, NULL, NULL, 0}, {"tiered_storage", "category", NULL, NULL, confchk_WT_SESSION_create_tiered_storage_subconfigs, - 6}, + 7}, {"tiers", "list", NULL, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, @@ -804,6 +806,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_statistics_log_subconfigs[] static const WT_CONFIG_CHECK confchk_tiered_storage_subconfigs[] = { {"auth_token", "string", NULL, NULL, NULL, 0}, {"bucket", "string", NULL, NULL, NULL, 0}, {"bucket_prefix", "string", NULL, NULL, NULL, 0}, + {"cache_directory", "string", NULL, NULL, NULL, 0}, {"local_retention", "int", NULL, "min=0,max=10000", NULL, 0}, {"name", "string", NULL, NULL, NULL, 0}, {"object_target_size", "int", NULL, "min=100K,max=10TB", NULL, 0}, @@ -864,7 +867,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, - {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 6}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 7}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," @@ -946,7 +949,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, - {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 6}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 7}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," @@ -1025,7 +1028,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, - {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 6}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 7}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," @@ -1102,7 +1105,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, - {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 6}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 7}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," @@ -1222,9 +1225,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "prefix_compression=false,prefix_compression_min=4,readonly=false" ",source=,split_deepen_min_child=0,split_deepen_per_child=0," "split_pct=90,tiered_object=false,tiered_storage=(auth_token=," - "bucket=,bucket_prefix=,local_retention=300,name=," - "object_target_size=10M),type=file,value_format=u,verbose=[]," - "write_timestamp_usage=none", + "bucket=,bucket_prefix=,cache_directory=,local_retention=300," + "name=,object_target_size=10M),type=file,value_format=u," + "verbose=[],write_timestamp_usage=none", confchk_WT_SESSION_create, 50}, {"WT_SESSION.drop", "checkpoint_wait=true,force=false,lock_wait=true," @@ -1289,8 +1292,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "prefix_compression=false,prefix_compression_min=4,readonly=false" ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," "tiered_object=false,tiered_storage=(auth_token=,bucket=," - "bucket_prefix=,local_retention=300,name=,object_target_size=10M)" - ",value_format=u,verbose=[],write_timestamp_usage=none", + "bucket_prefix=,cache_directory=,local_retention=300,name=," + "object_target_size=10M),value_format=u,verbose=[]," + "write_timestamp_usage=none", confchk_file_config, 42}, {"file.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," @@ -1309,8 +1313,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "prefix_compression_min=4,readonly=false,split_deepen_min_child=0" ",split_deepen_per_child=0,split_pct=90,tiered_object=false," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," - "local_retention=300,name=,object_target_size=10M),value_format=u" - ",verbose=[],version=(major=0,minor=0),write_timestamp_usage=none", + "cache_directory=,local_retention=300,name=," + "object_target_size=10M),value_format=u,verbose=[]," + "version=(major=0,minor=0),write_timestamp_usage=none", confchk_file_meta, 47}, {"index.meta", "app_metadata=,assert=(commit_timestamp=none," @@ -1340,8 +1345,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",readonly=false,split_deepen_min_child=0," "split_deepen_per_child=0,split_pct=90,tiered_object=false," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," - "local_retention=300,name=,object_target_size=10M),value_format=u" - ",verbose=[],write_timestamp_usage=none", + "cache_directory=,local_retention=300,name=," + "object_target_size=10M),value_format=u,verbose=[]," + "write_timestamp_usage=none", confchk_lsm_meta, 46}, {"object.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," @@ -1360,8 +1366,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "prefix_compression_min=4,readonly=false,split_deepen_min_child=0" ",split_deepen_per_child=0,split_pct=90,tiered_object=false," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," - "local_retention=300,name=,object_target_size=10M),value_format=u" - ",verbose=[],version=(major=0,minor=0),write_timestamp_usage=none", + "cache_directory=,local_retention=300,name=," + "object_target_size=10M),value_format=u,verbose=[]," + "version=(major=0,minor=0),write_timestamp_usage=none", confchk_object_meta, 48}, {"table.meta", "app_metadata=,assert=(commit_timestamp=none," @@ -1373,22 +1380,23 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," "assert=(commit_timestamp=none,durable_timestamp=none," "read_timestamp=none,write_timestamp=off),block_allocation=best," - "block_compressor=,bucket=,bucket_prefix=,cache_resident=false," - "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=,checksum=on," - "collator=,columns=,dictionary=0,encryption=(keyid=,name=)," - "format=btree,huffman_key=,huffman_value=,id=," - "ignore_in_memory_cache_size=false,internal_item_max=0," - "internal_key_max=0,internal_key_truncate=true," - "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," - "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," - "log=(enabled=true),memory_page_image_max=0,memory_page_max=5MB," - "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," - "prefix_compression_min=4,readonly=false,split_deepen_min_child=0" - ",split_deepen_per_child=0,split_pct=90,tiered_object=false," - "tiered_storage=(auth_token=,bucket=,bucket_prefix=," - "local_retention=300,name=,object_target_size=10M),value_format=u" - ",verbose=[],version=(major=0,minor=0),write_timestamp_usage=none", - confchk_tier_meta, 49}, + "block_compressor=,bucket=,bucket_prefix=,cache_directory=," + "cache_resident=false,checkpoint=,checkpoint_backup_info=," + "checkpoint_lsn=,checksum=on,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_key=," + "huffman_value=,id=,ignore_in_memory_cache_size=false," + "internal_item_max=0,internal_key_max=0," + "internal_key_truncate=true,internal_page_max=4KB,key_format=u," + "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," + "leaf_value_max=0,log=(enabled=true),memory_page_image_max=0," + "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," + "prefix_compression=false,prefix_compression_min=4,readonly=false" + ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "tiered_object=false,tiered_storage=(auth_token=,bucket=," + "bucket_prefix=,cache_directory=,local_retention=300,name=," + "object_target_size=10M),value_format=u,verbose=[]," + "version=(major=0,minor=0),write_timestamp_usage=none", + confchk_tier_meta, 50}, {"tiered.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," "assert=(commit_timestamp=none,durable_timestamp=none," @@ -1406,9 +1414,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "prefix_compression=false,prefix_compression_min=4,readonly=false" ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," "tiered_object=false,tiered_storage=(auth_token=,bucket=," - "bucket_prefix=,local_retention=300,name=,object_target_size=10M)" - ",tiers=,value_format=u,verbose=[],version=(major=0,minor=0)," - "write_timestamp_usage=none", + "bucket_prefix=,cache_directory=,local_retention=300,name=," + "object_target_size=10M),tiers=,value_format=u,verbose=[]," + "version=(major=0,minor=0),write_timestamp_usage=none", confchk_tiered_meta, 49}, {"wiredtiger_open", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" @@ -1440,10 +1448,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," - "local_retention=300,name=,object_target_size=10M)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),use_environment=true,use_environment_priv=false," - "verbose=[],verify_metadata=false,write_through=", + "cache_directory=,local_retention=300,name=," + "object_target_size=10M),timing_stress_for_test=," + "transaction_sync=(enabled=false,method=fsync)," + "use_environment=true,use_environment_priv=false,verbose=[]," + "verify_metadata=false,write_through=", confchk_wiredtiger_open, 57}, {"wiredtiger_open_all", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" @@ -1475,11 +1484,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," - "local_retention=300,name=,object_target_size=10M)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),use_environment=true,use_environment_priv=false," - "verbose=[],verify_metadata=false,version=(major=0,minor=0)," - "write_through=", + "cache_directory=,local_retention=300,name=," + "object_target_size=10M),timing_stress_for_test=," + "transaction_sync=(enabled=false,method=fsync)," + "use_environment=true,use_environment_priv=false,verbose=[]," + "verify_metadata=false,version=(major=0,minor=0),write_through=", confchk_wiredtiger_open_all, 58}, {"wiredtiger_open_basecfg", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" @@ -1511,10 +1520,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "tiered_manager=(threads_max=8,threads_min=1,wait=0)," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," - "local_retention=300,name=,object_target_size=10M)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=[],verify_metadata=false,version=(major=0," - "minor=0),write_through=", + "cache_directory=,local_retention=300,name=," + "object_target_size=10M),timing_stress_for_test=," + "transaction_sync=(enabled=false,method=fsync),verbose=[]," + "verify_metadata=false,version=(major=0,minor=0),write_through=", confchk_wiredtiger_open_basecfg, 52}, {"wiredtiger_open_usercfg", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" @@ -1546,9 +1555,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "tiered_manager=(threads_max=8,threads_min=1,wait=0)," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," - "local_retention=300,name=,object_target_size=10M)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=[],verify_metadata=false,write_through=", + "cache_directory=,local_retention=300,name=," + "object_target_size=10M),timing_stress_for_test=," + "transaction_sync=(enabled=false,method=fsync),verbose=[]," + "verify_metadata=false,write_through=", confchk_wiredtiger_open_usercfg, 51}, {NULL, NULL, NULL, 0}}; diff --git a/src/third_party/wiredtiger/src/config/test_config.c b/src/third_party/wiredtiger/src/config/test_config.c index 135cf6f01a2..30bf6d8a3c3 100644 --- a/src/third_party/wiredtiger/src/config/test_config.c +++ b/src/third_party/wiredtiger/src/config/test_config.c @@ -85,6 +85,20 @@ static const WT_CONFIG_CHECK confchk_base_test[] = { {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 2}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_burst_inserts[] = { + {"burst_duration", "string", NULL, NULL, NULL, 0}, + {"cache_size_mb", "int", NULL, "min=0,max=100000000000", NULL, 0}, + {"checkpoint_manager", "category", NULL, NULL, confchk_checkpoint_manager_subconfigs, 2}, + {"compression_enabled", "boolean", NULL, NULL, NULL, 0}, + {"duration_seconds", "int", NULL, "min=0,max=1000000", NULL, 0}, + {"enable_logging", "boolean", NULL, NULL, NULL, 0}, + {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 5}, + {"statistics_config", "category", NULL, NULL, confchk_statistics_config_subconfigs, 2}, + {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 4}, + {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 6}, + {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 2}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_example_test[] = { {"cache_size_mb", "int", NULL, "min=0,max=100000000000", NULL, 0}, {"checkpoint_manager", "category", NULL, NULL, confchk_checkpoint_manager_subconfigs, 2}, @@ -130,6 +144,25 @@ static const WT_CONFIG_ENTRY config_entries[] = { "min=0),thread_count=0,value_size=5))," "workload_tracking=(enabled=true,op_rate=1s)", confchk_base_test, 10}, + {"burst_inserts", + "burst_duration=90,cache_size_mb=0," + "checkpoint_manager=(enabled=false,op_rate=1s)," + "compression_enabled=false,duration_seconds=0," + "enable_logging=false,runtime_monitor=(enabled=true,op_rate=1s," + "postrun_statistics=[],stat_cache_size=(enabled=false,limit=0)," + "stat_db_size=(enabled=false,limit=0))," + "statistics_config=(enable_logging=true,type=all)," + "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s," + "stable_lag=1),workload_generator=(enabled=true," + "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," + "min=0),thread_count=0,value_size=5),op_rate=1s," + "populate_config=(collection_count=1,key_count_per_collection=0," + "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s," + "ops_per_transaction=(max=1,min=0),thread_count=0)," + "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," + "min=0),thread_count=0,value_size=5))," + "workload_tracking=(enabled=true,op_rate=1s)", + confchk_burst_inserts, 11}, {"example_test", "cache_size_mb=0,checkpoint_manager=(enabled=false,op_rate=1s)," "compression_enabled=false,duration_seconds=0," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 95ca5acbac5..c06ab898882 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -759,6 +759,11 @@ __wt_conn_remove_storage_source(WT_SESSION_IMPL *session) TAILQ_REMOVE(&nstorage->bucketqh, bstorage, q); __wt_free(session, bstorage->auth_token); __wt_free(session, bstorage->bucket); + __wt_free(session, bstorage->bucket_prefix); + __wt_free(session, bstorage->cache_directory); + if (bstorage->file_system != NULL && bstorage->file_system->terminate != NULL) + WT_TRET( + bstorage->file_system->terminate(bstorage->file_system, (WT_SESSION *)session)); __wt_free(session, bstorage); } diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 3abeeb91152..decf70dd7e4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -989,6 +989,9 @@ __wt_dhandle_update_write_gens(WT_SESSION_IMPL *session) WT_WITH_HANDLE_LIST_WRITE_LOCK(session, WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); if (dhandle == NULL) break; + /* There can be other dhandle types such as tier: that do not have a btree. Skip those. */ + if (!WT_BTREE_PREFIX(dhandle->name)) + continue; btree = (WT_BTREE *)dhandle->handle; WT_ASSERT(session, btree != NULL); diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 04e53e5b4f2..ce54fd23abf 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -30,10 +30,10 @@ __curfile_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) CURSOR_API_CALL(a, session, compare, CUR2BT(cbt)); /* - * Check both cursors are a "file:" type then call the underlying function, it can handle - * cursors pointing to different objects. + * Check both cursors are a btree type then call the underlying function, it can handle cursors + * pointing to different objects. */ - if (!WT_PREFIX_MATCH(a->internal_uri, "file:") || !WT_PREFIX_MATCH(b->internal_uri, "file:")) + if (!WT_BTREE_PREFIX(a->internal_uri) || !WT_BTREE_PREFIX(b->internal_uri)) WT_ERR_MSG(session, EINVAL, "Cursors must reference the same object"); WT_ERR(__cursor_checkkey(a)); @@ -60,10 +60,10 @@ __curfile_equals(WT_CURSOR *a, WT_CURSOR *b, int *equalp) CURSOR_API_CALL(a, session, equals, CUR2BT(cbt)); /* - * Check both cursors are a "file:" type then call the underlying function, it can handle - * cursors pointing to different objects. + * Check both cursors are a btree type then call the underlying function, it can handle cursors + * pointing to different objects. */ - if (!WT_PREFIX_MATCH(a->internal_uri, "file:") || !WT_PREFIX_MATCH(b->internal_uri, "file:")) + if (!WT_BTREE_PREFIX(a->internal_uri) || !WT_BTREE_PREFIX(b->internal_uri)) WT_ERR_MSG(session, EINVAL, "Cursors must reference the same object"); WT_ERR(__cursor_checkkey(a)); @@ -412,8 +412,11 @@ __curfile_remove(WT_CURSOR *cursor) __wt_stat_usecs_hist_incr_opwrite(session, WT_CLOCKDIFF_US(time_stop, time_start)); /* If we've lost an initial position, we must fail. */ - if (positioned && !F_ISSET(cursor, WT_CURSTD_KEY_INT)) + if (positioned && !F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + WT_IGNORE_RET(__wt_msg( + session, "WT_ROLLBACK: rolling back cursor remove as initial position was lost")); WT_ERR(WT_ROLLBACK); + } /* * Remove with a search-key is fire-and-forget, no position and no key. Remove starting from a @@ -821,7 +824,7 @@ __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, c if (bulk) LF_SET(WT_BTREE_BULK | WT_DHANDLE_EXCLUSIVE); - WT_ASSERT(session, WT_PREFIX_MATCH(uri, "file:") || WT_PREFIX_MATCH(uri, "tiered:")); + WT_ASSERT(session, WT_BTREE_PREFIX(uri)); /* Get the handle and lock it while the cursor is using it. */ /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index adacc1f3e68..776ad980d3f 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -861,7 +861,7 @@ __wt_cursor_cache_get(WT_SESSION_IMPL *session, const char *uri, uint64_t hash_v /* * If this is a btree cursor, clear its read_once flag. */ - if (WT_PREFIX_MATCH(cursor->internal_uri, "file:")) { + if (WT_BTREE_PREFIX(cursor->internal_uri)) { cbt = (WT_CURSOR_BTREE *)cursor; F_CLR(cbt, WT_CBT_READ_ONCE); } else { @@ -1020,6 +1020,29 @@ __cursor_config_debug(WT_CURSOR *cursor, const char *cfg[]) } /* + * __check_prefix_format -- + * Check if the schema format is a fixed-length string, variable string or byte array. + */ +static int +__check_prefix_format(const char *format) +{ + size_t len; + const char *p; + + /* Early exit if prefix format is S or u. */ + if (WT_STREQ(format, "S") || WT_STREQ(format, "u")) + return (0); + /* + * Now check for fixed-length string format through looking at the characters before the nul + * character. + */ + for (p = format, len = strlen(format); len > 1; --len, p++) + if (!__wt_isdigit((u_char)*p)) + break; + return ((len > 1 || *p != 's') ? EINVAL : 0); +} + +/* * __wt_cursor_reconfigure -- * Set runtime-configurable settings. */ @@ -1067,6 +1090,14 @@ __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) if (WT_CURSOR_RECNO(cursor)) WT_ERR_MSG( session, EINVAL, "cannot use prefix key search near for column store formats"); + /* + * Prefix search near configuration can only be used for string or raw byte array + * formats. + */ + if ((ret = __check_prefix_format(cursor->key_format)) != 0) + WT_ERR_MSG(session, ret, + "prefix key search near can only be used with string, fixed-length string or raw " + "byte array format types"); if (CUR2BT(cursor)->collator != NULL) WT_ERR_MSG( session, EINVAL, "cannot use prefix key search near with a custom collator"); diff --git a/src/third_party/wiredtiger/src/docs/arch-block.dox b/src/third_party/wiredtiger/src/docs/arch-block.dox index 1738d2fdb9b..d5f7b6dff1e 100644 --- a/src/third_party/wiredtiger/src/docs/arch-block.dox +++ b/src/third_party/wiredtiger/src/docs/arch-block.dox @@ -150,7 +150,7 @@ API is called. See \c bt_io.c for more detail. @subsubsection Checkpoint -For details on checkpoint at the WiredTiger level see: @ref arch-checkpoint. +For details on checkpoint at the WiredTiger level see: @ref arch-checkpoint "Checkpoint". At the block manager level, a checkpoint corresponds with a set of blocks that are stored in the file associated with the given URI. Typically each file will @@ -223,7 +223,7 @@ range in the file, that is, any number of contiguous blocks. @subsubsection Eviction -For more detail on how WiredTiger eviction works see: @ref arch-eviction. +For more detail on how WiredTiger eviction works see: @ref arch-eviction "Eviction". Eviction also utilizes the block manager. When a page is evicted and contains data that needs to be maintained, logically a block needs to be written. diff --git a/src/third_party/wiredtiger/src/docs/arch-btree.dox b/src/third_party/wiredtiger/src/docs/arch-btree.dox new file mode 100644 index 00000000000..c416ff22759 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/arch-btree.dox @@ -0,0 +1,87 @@ +/*! @arch_page arch-btree B-Trees + +WiredTiger represents database tables using a B-Tree data structure (\c WT_BTREE in +\c btree.h), which is made up of nodes that are page structures. The root and internal +pages only store keys and references to other pages, while leaf pages store keys and +values. Pages are populated with records as users insert data into the database. Records +are maintained in sorted order according to their key, and pages will split once their +configured limit is reached, causing the B-Tree to expand. Pages have an in-memory +representation as well as an on-disk representation. The focus here will be on +the in-memory representation of the B-Tree and its pages as defined in \c btmem.h. The +on-disk representation is discussed in the @ref arch-data-file page. + +@section btree_btree_data_source B-Tree Data Source (WT_BTREE) + +As discussed in the @ref arch-dhandle page, data handles (dhandles), are generic +containers used to access various types of data sources. Dhandles that represent +B-Trees contain a pointer to the \c WT_BTREE. At a high-level, the \c WT_BTREE +contains a memory cache of key value pairs, along with functions to read and +write data as needed to and from the data file. It also contains the specific WT_BTREE +type, a reference to the root page on disk for access to the underlying data, and +information used to maintain the B-Tree structure. The different B-Tree types support +different access methods; row-store is most commonly used (\c WT_BTREE_ROW), and +column-store is available with either fixed-length records (\c WT_BTREE_COL_FIX) or +variable-length records (\c WT_BTREE_COL_VAR) (see @ref arch-row-column for more details). + +@section btree_btree_in_memory_representation B-Tree In-Memory Representation + +B-Trees can grow to a very large size, and the space in memory is +generally not large enough to hold all the pages of the B-Tree. To access a page in +the B-Tree, we require a \c WT_REF which tracks whether the page has or has not been +loaded from storage. Once the page is loaded, the \c WT_REF or reference structure will +have a valid \c WT_PAGE pointer which represents the in-memory page. The \c WT_BTREE +structure contains a \c WT_REF that points to the root page of the given tree. Other +pages can be accessed as required by traversing through the child structures of the root +page, and the child structures of those pages, and so on. + +To insert or modify values in a row-store B-Tree, the code traverses down to the leaf +pages which contain the key/value pairs (\c WT_ROW structure). New key/value pairs are +inserted into row-store leaf pages using a \c WT_INSERT structure. Existing entries on +leaf pages can be updated, modified or deleted through \c WT_UPDATE structures. As new +updates are made, these structures are chained into an update list. This means that an +entry may have some old values, or deleted values, which may be visible depending on +the timestamp used by a reader. + +@section btree_truncate_operation Truncate Operation + +Truncate allows multiple records in a specified range to be deleted in a single operation. +It is much faster and more efficient than deleting each record individually. Fast-truncate +is an optimization that WiredTiger makes internally; whole pages are marked as deleted +without having to first instantiate the page in memory or inspect records individually. In +situations where this is not possible, a slower truncate will walk keys individually, putting +a tombstone onto each one to mark deletion. Truncation is also possible for log files but +the focus here will be on truncation for B-Tree data files (\c file: uri). + +@section btree_range_truncate_file Range Truncate On Files + +To perform a truncate on a file, a start and stop cursor are positioned corresponding +to the desired range provided by the user. The desired start and stop keys do not actually +need to exist since the cursors are positioned by doing a search-near rather than a search. +Once positioned, we do a page-by-page walk on the B-Tree, fast-truncating pages where +possible. When a page is marked for fast-truncation, its \c WT_READ_TRUNCATE flag is set, +and the code tries to delete the page without instantiating it into memory. If no errors +are encountered, a page-deleted structure \c WT_PAGE_DELETED is allocated and initialized +with the timestamp, transaction id and state information associated with the truncate. +Finally, the page is published as deleted using the \c WT_REF_DELETED flag. Pages which are +not eligible for fast truncation are pages where only part of it will be deleted (e.g. the +first and last pages of the truncate range), pages with overflow items, or pages with +prepared updates. These pages will have their records deleted individually. + +@section btree_truncate_interaction_with_other_operations Interaction With Other Operations + +Truncation differs from other operations in the sense that it does not have the same +transactional semantics. If a range truncate is in progress and another transaction +happens to insert a key into the same range, the behavior is not well-defined. A +conflict may be detected, or both transaction may be permitted to commit. In the scenario +that both commit, if the system crashes and recovery runs, the resulting state of the +database may be different to what was in the cache had the crash not happened. + +@section btree_truncate_performing_reads_on_truncated_pages Performing Reads On Truncated Pages + +In some scenarios, a reader may want to read older values from a page from a point in time +before the page got truncated. The page must first be re-instantiated into memory; the +\c __wt_page_in_func (see \c bt_read.c) reads in a page from disk, and builds an +in-memory version. Then as part of the page read process, we create a \c WT_UPDATE +with a tombstone in the same transaction the truncate happened. + +*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-connection.dox b/src/third_party/wiredtiger/src/docs/arch-connection.dox index cdc3eeecb9a..4b63bd02dfe 100644 --- a/src/third_party/wiredtiger/src/docs/arch-connection.dox +++ b/src/third_party/wiredtiger/src/docs/arch-connection.dox @@ -13,7 +13,8 @@ connection is represented by \c WT_CONNECTION. A connection is initialized when WT_CONNECTION::wiredtiger_open is called by the user application. WT_CONNECTION::wiredtiger_open accepts a list of configuration items (see @ref database_config) that can be used to enable different WiredTiger features and tune their behavior. Those features are for -example related to @ref arch-eviction, @ref arch-logging, @ref arch-checkpoint, @ref arch-cache, +example related to @ref arch-eviction "Eviction", @ref arch-logging "Logging", +@ref arch-checkpoint "Checkpoints", @ref arch-cache "Caching", statistics, etc. All the different available configuration settings are described in the documentation for WT_CONNECTION::wiredtiger_open. @@ -32,17 +33,17 @@ database. Once the database has been successfully opened, internal worker threads are started to provide global services used at runtime. Those services consist of different threads to handle statistics, logging, eviction, checkpoint and cache management. The sweeping server that manages the active and -inactive dhandles is started too, see @ref arch-dhandle for more information. +inactive dhandles is started too, see @ref arch-dhandle "Data Handles" for more information. Finally, before the connection is completely initialized, the database is set to a consistent state -by running rollback to stable, see @ref arch-rts for more details. +by running @ref arch-rts "rollback to stable". @subsection arch_conn_runtime Runtime At runtime, database-wide operations can be executed using the connection interface. For instance, it is possible to reconfigure WiredTiger features and behavior using WT_CONNECTION::reconfigure instead of closing the connection and calling WT_CONNECTION::open again. However, almost all CRUD -operations on the database are executed in the context of a session (see @ref arch-session) which +operations on the database are executed in the context of a @ref arch-session "Session" which can be created using WT_CONNECTION::open_session. See the WT_CONNECTION:: documentation to discover other available APIs related to WiredTiger connections. diff --git a/src/third_party/wiredtiger/src/docs/arch-cursor.dox b/src/third_party/wiredtiger/src/docs/arch-cursor.dox index 7eee860af86..4b29d8f39b4 100644 --- a/src/third_party/wiredtiger/src/docs/arch-cursor.dox +++ b/src/third_party/wiredtiger/src/docs/arch-cursor.dox @@ -146,7 +146,7 @@ With projections, the simple algorithm following the plan works well if the colu are grouped by column group and requested in order. Without that discipline, as in this example, the performance will not be optimal. -The implementation of plan creation and execution resides in the @ref arch-schema. +The implementation of plan creation and execution resides in the @ref arch-schema "Schema". @section arch_cursor_dump Dump cursors diff --git a/src/third_party/wiredtiger/src/docs/arch-dhandle-lifecycle.dox b/src/third_party/wiredtiger/src/docs/arch-dhandle-lifecycle.dox deleted file mode 100644 index 0a151eb41bc..00000000000 --- a/src/third_party/wiredtiger/src/docs/arch-dhandle-lifecycle.dox +++ /dev/null @@ -1,92 +0,0 @@ -/*! @page arch-dhandle-lifecycle Data Handle Lifecycle - -A WiredTiger Data Handle (dhandle) is a generic representation of any named -data source. This representation contains information such as its name, -how many references there are to it, individual data source statistics and -what type of underlying data object it is. - -WiredTiger maintains all dhandles in a global dhandle list accessed -from the connection. Multiple sessions access this list, which is -protected by a R/W lock. Each session also maintains a session -dhandle cache which is a cache of dhandles a session has operated -upon. The entries in the cache are references into the global dhandle -list. - -@section dhandle-creation dhandle creation - -When a cursor in a session attempts to access a WiredTiger table -that has not been accessed before, the dhandle for the table will -neither be in the session's dhandle cache nor in the connection's -global dhandle list. A new dhandle will have to be created for this -table. The cursor trying to access a table first attempts to find -the dhandle in the session dhandle cache. When it doesn't find the -dhandle in the session cache, it searches in the global dhandle -list while holding the read lock. When it doesn't find the dhandle -there, it creates a dhandle for this table and puts it in the global -dhandle list while holding the write lock on the global dhandle -list. The cursor operation then puts a reference to the dhandle in -the session's dhandle cache. - -There are two relevant reference counters in the dhandle structure, -\c session_ref and \c session_inuse. \c session_ref counts the -number of session dhandle cache lists that this contain dhandle. -\c session_inuse is a count of cursors opened and operating on this -dhandle. Both these counters are incremented by this session as the -cursor attempts to use this dhandle. When the operation completes -and the cursor is closed, the \c session_inuse is decremented. The -dhandle reference is not removed immediately from the session dhandle -cache. If this session accesses the same table again in the near -future, having the dhandle reference already in the session dhandle -cache is a performance optimization. - -@section session-cache-sweep dhandle session cache sweep - -Dhandle cache sweep is the only way a cleanup is performed on the -session' dhandle cache list. The references to the dhandles that have not -been accessed by this session in a long time are removed from the -cache. Since a session is single-threaded, a session's dhandle cache -can only be altered by that session alone. - -Each time a session accesses a dhandle, it checks if enough -time has elapsed to do a session cache sweep for that session. -As it walks the session dhandle cache list, it notices if any dhandle -on its list has been marked dead (idle too long). If it has, the -session removes that dhandle from its list and decrements the -\c session_ref count. - -Since accessing a dhandle involves walking the session dhandle cache -list anyway, cache cleanup is piggy-backed on this operation. - -@section dhandle-sweep sweep-server dhandle sweep - -WiredTiger maintains a sweep server in the background for the cleanup of the -global dhandle list. The sweep server periodically (\c close_scan_interval) -revisits the dhandles in the global list and if the dhandles are not being used, -i.e., the \c session_inuse count is 0, it assigns the current time as the time -of death for the dhandle, if not already done before. - -If a dhandle has already got a time of death set for it in a previous iteration -of the dhandle sweep and the dhandle has stayed not in use, the sweep server -compares the time of death with the current time to check if the dhandle has -remained idle for the configured idle time (\c close_idle_time). If the dhandle -has remained idle, the sweep server closes the associated btree contained in the -dhandle and releases some of the resources for that dhandle. It also marks -the dhandle as dead so that the next time a session with a reference walks its -own cache list, it will see the handle marked dead and remove it from the -session's dhandle cache list (see above). - -The sweep server then checks for whether or not any session is referencing this -dhandle, i.e., if a session's dhandle cache still contains a reference to this -dhandle. If a dhandle stays referenced by at least one session, i.e., the -\c session_ref count is not 0, the dhandle can't be removed from the global -list. If this dhandle is not referenced by any session, i.e., \c session_ref -count is 0, the sweep server removes the dhandle from the global dhandle list -and frees any remaining resources associated with it. The removal of the dhandle -from the global list hence completes this dhandle's lifecycle. Any future access -of the associated table would need to start by creating the dhandle again. - -Note: The sweep server's scan interval and a dhandle's close idle time can be -configured using \c file_manager configuration settings of the connection -handle. - -*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-dhandle.dox b/src/third_party/wiredtiger/src/docs/arch-dhandle.dox index 118285ec014..50b00484ec1 100644 --- a/src/third_party/wiredtiger/src/docs/arch-dhandle.dox +++ b/src/third_party/wiredtiger/src/docs/arch-dhandle.dox @@ -1,17 +1,85 @@ -/*! @arch_page arch-dhandle Data Handles and Btrees +/*! @arch_page arch-dhandle Data Handles -An internal structure called a Data Handle (dhandle) is used to represent and -access Btrees and other data sources in WiredTiger. A dhandle is created -when a table is accessed for the first time. It is kept in a global list -and is shared across the sessions. When a dhandle is not needed anymore -and has been idle for a while, it is closed and destroyed, releasing all the -resources associated with it. +The data handle (dhandle) (\c WT_DATA_HANDLE) is a generic representation of +any named data source. Dhandles are required to access any data source in the +system. For the dhandle of B-Tree type, the \c void *handle field can be used +like so `(WT_BTREE *)dhandle->handle` to get the B-Tree. For other data sources +such as the table, LSM or tiered storage types, a type cast can be used on the +\c dhandle itself to gain access to the structure for the data source. The +dhandle structure also contains a set of flags, some reference counts, and +individual data source statistics. -A Btree is one kind of dhandle. It embodies both the on-disk and in-memory -representations of the Btree. +@section dhandle_data_handle_lifecycle Data Handle Lifecycle -@subpage arch-dhandle-lifecycle +Dhandles are owned by the connection and shared among sessions. WiredTiger +maintains all dhandles in a global dhandle list accessed from the connection. +Multiple sessions access this list, which is protected by a R/W lock. Each +session also maintains a session dhandle cache which is a cache of dhandles +a session has operated upon. The entries in the cache are references into +the global dhandle list. -This section describes how dhandles are created, and how and when they are destroyed. +@section dhandle_data_handle_creation Data Handle Creation + +Generally, dhandles are created when accessing tables and other data sources +that have not been accessed before. When a cursor in a session is opened on +a WiredTiger table, it first attempts to find the dhandle in the session +dhandle cache. If the dhandle is not found in the session dhandle cache, it +searches the global dhandle list while holding the read lock. In the case +that it doesn't find the dhandle there, it creates a dhandle for this table +and puts it in the global dhandle list while holding the write lock on the +global dhandle list. Finally, the cursor operation puts a reference to +the dhandle in the session's dhandle cache. + +@section dhandle_data_handle_reference_counts Data Handle Reference Counts + +There are two relevant reference counters in the dhandle structure, +\c session_ref and \c session_inuse. \c session_ref counts the number of +session dhandle cache lists that contain this dhandle. \c session_inuse +is a count of the number of cursors opened and operating on this dhandle. +Both these counters are incremented by the session as the cursor is opened on +this dhandle. \c session_inuse is decremented when the operation completes +and the cursor is closed. + +@section dhandle_sweep_server_dhandle_sweep Sweep-Server Dhandle Sweep + +WiredTiger maintains a sweep server in the background for the cleanup of +the global dhandle list. As dictated by the \c close_scan_interval +variable, the sweep server periodically revisits the dhandles in the +global list and if a dhandle is no longer being used (signified by the +\c session_inuse count going to 0), it assigns the current time as the +time of death if not already done before. + +Dhandles are not immediately closed when the time of death field is set. +The sweep server will keep the dhandle there for a configured amount of +idle time (\c close_idle_time), and will only close the associated data +source if it remained idle for that amount of time since time of death. +The closure of the dhandle often occurs on the next iteration of the +sweep when the sweep server detects the dhandle has remained idle for +long enough. The server marks the dhandle as dead so that the next time +a session with a reference walks its own cache list, it will see the +handle marked dead and remove it from the session's dhandle cache list. + +The sweep server then checks for any sessions that are referencing this +dhandle. If a dhandle stays referenced by at least one session +(\c session_ref count > 0), the dhandle cannot be removed from the +global list. If the dhandle is not referenced by any session, the sweep +server removes the dhandle from the global dhandle list and frees any +remaining resources associated with it. The removal of the dhandle from +the global list completes the lifecycle, and any future access of the +associated table will require the dhandle to be created again. + +@section dhandle_dhandle_session_cache_sweep Dhandle Session Cache Sweep + +A session's dhandle cache list is periodically cleaned out through a +dhandle cache sweep, which removes references to the dhandles that +have not been accessed by the session in a long time. Sessions have +their own sweeps; since a session is single-threaded, a session's +dhandle cache can only be altered by that session alone. + +Each time a session accesses a dhandle, it checks if enough time has +elapsed to do a session cache sweep for that session. As it walks the +session dhandle cache list, it notices if any dhandle on its list has +been marked dead (idle for too long). If it has, the session removes +that dhandle from its list and decrements the session_ref count. */ diff --git a/src/third_party/wiredtiger/src/docs/arch-eviction.dox b/src/third_party/wiredtiger/src/docs/arch-eviction.dox index 5c877e6322f..0e3bf023be6 100644 --- a/src/third_party/wiredtiger/src/docs/arch-eviction.dox +++ b/src/third_party/wiredtiger/src/docs/arch-eviction.dox @@ -1,15 +1,86 @@ /*! @arch_page arch-eviction Eviction -Eviction represents the process or removing old data from the cache, -writing it to disk if it is dirty. The general strategy uses a dedicated -set of eviction threads that are tasked with identifying candidate data. -If the data needs to be written, it is reconciled (converting the -in-memory format to on-disk format), and then written. Clean memory -can be freed if needed. +@section eviction_overall Eviction -Eviction cannot be triggered directly by API calls, but happens as a result -of enough data being dirtied. +The goal of eviction is to manage the @ref arch-cache and keep the cache in the margin set by the +user. There is an upper limit for the total content and dirty content, whenever the cache +goes above the target then eviction gets triggered to bring it down under the limits. Eviction is managed using \c WT_EVICT_QUEUE structures, each of which contains a list of \c WT_EVICT_ENTRY structures. + +Eviction is composed of one eviction server, zero or more eviction worker threads and three +shared eviction_queues (two ordinary queues and one urgent queue). The eviction server's +goal is to find the pages that can be evicted. It has to walk the pages in a tree, one tree +after the other, and find evictable candidates. It makes fair judgments between +the trees, remembers where it was in a tree, and proceeds to the next tree from there. It +extracts all the pages that can be evicted, sorts them based on when they were accessed last, and +takes one-third oldest of the evictable candidates. This simulates an approximate Least Recently +Used (LRU) algorithm, as the references to these pages are then pushed into the eviction_queues. +The whole goal is to remove the content which is older to be evicted and put it in the queue. A user +can configure a minimum and a maximum number of eviction worker threads to be created through +the connection configuration string. The goal of eviction worker threads is to pop the +pages from the queues and evict them. Eviction worker threads wait for queues to be populated, +working with the eviction server through a CV signaling mechanism. They then get to evicting the +pages independently to the server, till the eviction queues are not empty. There is +a mechanism where the number of worker threads can be scaled dynamically between the provided range. +With this mechanism, the system automatically adjusts the available worker threads to an optimum +number that gives a good eviction performance, while minimizing the overhead of having a lot of +eviction threads. + +It is possible to have just the eviction server and no worker threads. In such a case, +the eviction server walks the tree to find pages to evict and evict these pages from the +cache. Pages that are being marked for forced eviction go onto the urgent +queue, and these pages take precedence over the pages from the ordinary eviction queues. + +If there are other threads reading the content, the page cannot be evicted and the +eviction server/worker threads check for that. Eviction has to lock and get exclusive +access to the page so that after having checked if the page is evictable and starting to +evict, a parallel reader doesn't start to access it. + +@section clean_dirty_data Clean vs dirty data +- Clean refers to data in the cache that is identical to the version stored on disk. +- Dirty refers to data in the cache that has been modified and needs to be reconciled + with the data stored on the disk. + +@section clean_dirty_eviction Clean vs dirty eviction +- In the case of clean eviction, there is no dirty content on the page and it is directly + removed from the memory. After that, the page remains solely on the disk unchanged. +- In the case of dirty eviction, the dirty page goes through reconciliation. The obsolete + content on the page is thrown out, the latest value goes to the data store and older + values go to the history store. + +Sometimes the workload generates enough new data that eviction is unable to reduce the cache content +and more new content is put into cache that can be removed by eviction. In such a case application +threads are also forced to do the work of eviction workers threads, before they can read or write more data. + +Eviction triggers and targets + +WiredTiger provides several configuration options for tuning how pages are evicted from +the cache. + +WiredTiger eviction tuning options can be configured when first opening +a database via ::wiredtiger_open, or changed after open with +WT_CONNECTION::reconfigure. + +The \c eviction_target configuration value is the level at +which WiredTiger attempts to keep the overall cache usage. Eviction worker +threads are active when the cache contains at least this much content, +expressed as a percentage of the total cache size. + +The \c eviction_trigger configuration value is the level at which application threads +start to perform the eviction. + +The \c eviction_dirty_target and \c eviction_dirty_trigger operate in a similar way to +the overall targets but only apply to dirty data in cache. In particular, application +threads will be throttled if the percentage of dirty data reaches the +\c eviction_dirty_trigger. Any page that has been modified since it was read from disk is +considered dirty. + +The \c eviction_updates_target perform eviction in worker threads when the cache contain +at least this many bytes of updates + +The \c eviction_updates_trigger trigger application threads to perform eviction when the +cache contains at least this many bytes of updates. + */ diff --git a/src/third_party/wiredtiger/src/docs/arch-fs-os.dox b/src/third_party/wiredtiger/src/docs/arch-fs-os.dox index cedade13248..e9e71dc9f14 100644 --- a/src/third_party/wiredtiger/src/docs/arch-fs-os.dox +++ b/src/third_party/wiredtiger/src/docs/arch-fs-os.dox @@ -1,10 +1,61 @@ -/*! @arch_page arch-fs-os File System and Operating System Interface +/*! @arch_page arch-fs-os File System Interface and Operating System Support -Internally, a layer of abstraction is above all operating system calls, -allowing main line WiredTiger code to make a call to single set of interfaces. -There are currently OS specific APIs for POSIX and Windows. +@section multiple_os Support for Multiple Operating Systems +WiredTiger runs on a variety of systems with different interfaces for file system +and operating system services. Therefore we have abstract implementations of various +OS services, such as threading, clock, etc. We choose which implementation of the +abstraction to include at compile time, and the rest of WT code doesn't need to worry +about these details. Currently WiredTiger supports POSIX and Windows operating systems +through maintaining separate folders for each system. By checking the current machine's +operating system, WiredTiger either compiles the \c os_posix or \c os_win folder. +Additional structures and definitions specific to the operating system are defined +in the header files \c posix.h or \c os_windows.h. -Additionally, a program can use WT_CONNECTION::set_file_system to register -a set of functions that will be called for each file system operation. -WiredTiger will then call these functions at the appropriate time. +@section file_system_and_handle WiredTiger File System and File Handle Interfaces +WiredTiger provides file system and file handle abstraction layers, or interfaces, +to accommodate for a standard set of functionalities across multiple file systems +that may not have identical features. + +@subsection file_system File System Interface +The file system interface handles system calls on the file system namespace such +as creation and deletion. WiredTiger defines the file system interface through +\c WT_FILE_SYSTEM, which is initialized when a WiredTiger connection is opened. The +interface has a set of function pointers that represent the file system functionalities +that WiredTiger supports for a directory. WiredTiger defines POSIX and Windows file +system function implementations in \c os_fs.c. + +@subsection file_handle File Handle Interface +WiredTiger has a file handle interface called WT_FILE_HANDLE which defines a +set of function pointers that represent I/O operations for a file. We use this +interface as a base class to create file handle implementations specific to the +operating system, for example \c WT_FILE_HANDLE_POSIX or \c WT_FILE_HANDLE_WINDOWS. +These WiredTiger file handle structures are initialized when WT_FILE_SYSTEM::fs_open_file +is called. + +In WiredTiger, whenever a file is opened a file handle is created to accommodate +the state of the file in cache. To prevent multiple file handles being created for a +file, WiredTiger uses a hash table to maintain an single file handle for each opened +file. A structure called \c WT_FH is used for accessing each entry in the hash table. +Thus WiredTiger checks the hash table via \c WT_FH for an open file handle. + +@subsection file_customization File System and Handle Customization +WiredTiger provides an API that allows programs to register their own customized file +system using the WT_CONNECTION::set_file_system function. Programs need to supply +their own file system structure and file handle implementations. See @ref custom_file_systems +for an example of how file system and handles are customized. + +@subsection file_in_memory In-memory support +The in-memory configuration changes WiredTiger to run only in cache with no disk writes +and is enabled through the WiredTiger connection configuration string. WiredTiger maintains +each file through providing an in memory buffer for each file handle. Both in-memory file +system and file handle implementations are found in \c os_fs_inmemory.c. + +@subsection file_stream File Stream +WiredTiger contains a file stream implementation called \c WT_FSTREAM. This structure +is mainly used for non-critical functionalities such as writing verbose messages or +writing to WiredTiger statistics logs. + +@section filesystem_usage Multiple System Architecture Support +WiredTiger supports x86_64, i386, mips, ppc, aarch64, s390x, and sparc system architectures +which are found in gcc.h. */ diff --git a/src/third_party/wiredtiger/src/docs/arch-hs.dox b/src/third_party/wiredtiger/src/docs/arch-hs.dox index c6453108f03..862ce7442d0 100644 --- a/src/third_party/wiredtiger/src/docs/arch-hs.dox +++ b/src/third_party/wiredtiger/src/docs/arch-hs.dox @@ -42,7 +42,8 @@ possible timestamp (\c WT_TS_MAX). The stop timestamp of the latest update in th updated once the prepared update is resolved. A tombstone becomes globally visible if the stop timestamp is less than the oldest timestamp and the stop transaction id is visible to all concurrent transactions in the system. A checkpoint can delete history store pages that only contain globally -visible tombstones. The garbage collection process is discussed in @ref arch-checkpoint. +visible tombstones. The garbage collection process is discussed in @ref arch-checkpoint +"Checkpoint". @section arch_hs_initialize History store initialization WiredTiger checks for an existing history store table at startup and creates a new table if it is @@ -237,7 +238,7 @@ created without records which have globally visible tombstones. This ensures tha only keeping the relevant historical data required to serve the oldest reader in the system or as dictated by the oldest timestamp set by the application. Once all records on a page are obsolete, the page itself can be removed to reduce the size of the history store table -(see @ref arch-transaction for more details about oldest timestamp). +(see @ref arch-transaction "Transaction" for more details about oldest timestamp). @section arch_hs_read Searching for older versions of a key in History Store When looking for an update visible to the current transaction, WiredTiger first searches the update @@ -251,13 +252,13 @@ in the history store for a key and read timestamp combination, each update would visibility based on transaction and timestamp based visibility rules. In case there are no records visible in the history store table, \c WT_NOTFOUND error is returned to the reader. -@section arch_hs_rts History store and rollback-to-stable -Rollback-to-stable searches the history store table for valid updates that are stable according to +@section arch_hs_rts History store and rollback to stable +@ref arch-rts "Rollback to stable" searches the history store table for valid updates that are +stable according to the supplied stable timestamp and replaces the on-disk version with an update from the history store -table that fulfills the criteria. Rollback-to-stable also removes updates from the +table that fulfills the criteria. Rollback to stable also removes updates from the history store table that are not stable according to the stable timestamp and transaction snapshot -being used for rollback-to-stable operation. More details about the rollback-to-stable operation -can be found in the @ref arch-rts page. +being used for rollback to stable operation. @section arch_hs_prepared History store and prepared transactions When there is a prepared update for a key and the page is evicted, the prepared update is written diff --git a/src/third_party/wiredtiger/src/docs/arch-index.dox b/src/third_party/wiredtiger/src/docs/arch-index.dox index 906e6959be8..1a8f252fe0a 100644 --- a/src/third_party/wiredtiger/src/docs/arch-index.dox +++ b/src/third_party/wiredtiger/src/docs/arch-index.dox @@ -122,94 +122,97 @@ wt_file -[hidden]right-> log_file We go into some detail for some of the internal components. -@subpage arch-backup +@subpage arch-backup "Backup" Hot backup uses a type of cursor to backup the database. -@subpage arch-block +@subpage arch-block "Block Manager" The Block Manager manages the reading and writing of disk blocks. +@subpage arch-btree +A B-Tree is one type of underlying data source in a dhandle and is organized into pages. + @subpage arch-cache Cache is represented by the various shared data structures that make up in-memory Btrees and subordinate data structures. -@subpage arch-checkpoint +@subpage arch-checkpoint "Checkpoint" A checkpoint is created by WiredTiger to serve as a point from which it can recover. -@subpage arch-connection +@subpage arch-connection "Connection" A connection is a handle to a WiredTiger database instance. -@subpage arch-cursor +@subpage arch-cursor "Cursor" Cursors are used to get and modify data. -@subpage arch-data-file +@subpage arch-data-file "Data File Format" The format of the data file is given by structures in \c block.h . -@subpage arch-dhandle +@subpage arch-dhandle "Data Handles" An internal structure called a Data Handle (dhandle) is used to represent and access Btrees and other data sources in WiredTiger. -@subpage arch-eviction +@subpage arch-eviction "Eviction" Eviction represents the process or removing old data from the cache, writing it to disk if it is dirty. -@subpage arch-fs-os +@subpage arch-fs-os "File System and Operating System Interface" A layer of abstraction is above all operating system calls and a set of functions can be registered to be called for each file system operation. -@subpage arch-hs +@subpage arch-hs "History Store" The History Store tracks old versions of records. -@subpage arch-logging +@subpage arch-logging "Logging" WiredTiger writes all changes into a write-ahead log when configured. -@subpage arch-log-file +@subpage arch-log-file "Log File Format" The format of a log file is defined in \c log.h . -@subpage arch-metadata +@subpage arch-metadata "Metadata" Metadata is stored as <code>uri, config</code> K/V pairs in a designated table. -@subpage arch-python +@subpage arch-python "Python API" WiredTiger has a Python API that is useful for scripting and experimentation. -@subpage arch-row-column +@subpage arch-row-column "Row Store and Column Store" Row stores and column store are B-Trees. Row stores have a variable size key and data while column stores have as their key a record id. -@subpage arch-rts +@subpage arch-rts "Rollback to Stable" Rollback to stable to remove the unstable updates from the database. -@subpage arch-schema +@subpage arch-schema "Schema" A schema defines the format of the application data in WiredTiger. -@subpage arch-session +@subpage arch-session "Session" A session defines the context for most operations performed in WiredTiger. -@subpage arch-snapshot +@subpage arch-snapshot "Snapshot" Snapshots are implemented by storing transaction ids committed before the transaction started. -@subpage arch-transaction +@subpage arch-transaction "Transactions" Transactions provide a powerful abstraction for multiple threads to operate on data concurrently. diff --git a/src/third_party/wiredtiger/src/docs/arch-log-file.dox b/src/third_party/wiredtiger/src/docs/arch-log-file.dox index 1ff828052db..a4d544d595f 100644 --- a/src/third_party/wiredtiger/src/docs/arch-log-file.dox +++ b/src/third_party/wiredtiger/src/docs/arch-log-file.dox @@ -1,7 +1,7 @@ /*! @arch_page arch-log-file Log File Format This page assumes familiarity with the logging subsystem. Please see -@ref arch-logging for the logging subsystem architecture description. +@ref arch-logging "Logging" for a description of that subsystem. Each log file begins with a fixed length header, followed by a set of variable length log records. The fixed length header contains information diff --git a/src/third_party/wiredtiger/src/docs/arch-logging.dox b/src/third_party/wiredtiger/src/docs/arch-logging.dox index aea99aa094f..fe36a9c6589 100644 --- a/src/third_party/wiredtiger/src/docs/arch-logging.dox +++ b/src/third_party/wiredtiger/src/docs/arch-logging.dox @@ -167,8 +167,7 @@ in the case of crash or failure. When the system is restarted, WiredTiger will a run recovery. If the system previously shutdown cleanly, then recovery determines that the log does not need to be replayed. Otherwise, recovery replays all operations from the LSN of the most recent checkpoint. When timestamps are in use, recovery also -performs a rollback-to-stable operation on tables that are not logged. -This aspect of recovery is discussed in @ref arch-rts. +performs a @ref arch-rts "rollback to stable" operation on tables that are not logged. Users can access and view the records in the log using one of two methods. The most common method is via the \c wt printlog command. The command utility will print the diff --git a/src/third_party/wiredtiger/src/docs/arch-row-column.dox b/src/third_party/wiredtiger/src/docs/arch-row-column.dox index d7d10f54795..affd8ea3e16 100644 --- a/src/third_party/wiredtiger/src/docs/arch-row-column.dox +++ b/src/third_party/wiredtiger/src/docs/arch-row-column.dox @@ -1,4 +1,4 @@ -/*! @arch_page arch-row-column Row Store & Column Store +/*! @arch_page arch-row-column Row Store and Column Store @section row_column_definition Definition @@ -14,8 +14,8 @@ column store is created. The fixed length column store has specialized use cases - The variable length is a more general case which allows for values to have any length similarly to row-stores. -Please refer to @ref arch-data-file to learn more about the on-disk format for row-stores and -column-stores. +Please refer to @ref arch-data-file "Data File Format" to learn more about the on-disk format for +row-stores and column-stores. @section row_column_rs Row-stores diff --git a/src/third_party/wiredtiger/src/docs/arch-rts.dox b/src/third_party/wiredtiger/src/docs/arch-rts.dox index 3e2cf6b70d0..7e469a5979a 100644 --- a/src/third_party/wiredtiger/src/docs/arch-rts.dox +++ b/src/third_party/wiredtiger/src/docs/arch-rts.dox @@ -1,4 +1,4 @@ -/*! @arch_page arch-rts Rollback to stable +/*! @arch_page arch-rts Rollback to Stable Rollback to stable is an operation that retains only the modifications that are stable according to the stable timestamp and recovered checkpoint snapshot. @@ -8,7 +8,7 @@ checkpoint are recovered and used as a recovered checkpoint snapshot. @section rts-overview Overview of rollback to stable Rollback to stable scans each and every table present in the database except -metadata @ref arch-metadata to remove the modifications from the table +@ref arch-metadata "Metadata" to remove the modifications from the table that are more than stable timestamp and recovered checkpoint snapshot. In the process of removing newer modifications from the table, all the in-memory diff --git a/src/third_party/wiredtiger/src/docs/arch-schema-ops.dox b/src/third_party/wiredtiger/src/docs/arch-schema-ops.dox deleted file mode 100644 index 5d647a05205..00000000000 --- a/src/third_party/wiredtiger/src/docs/arch-schema-ops.dox +++ /dev/null @@ -1,206 +0,0 @@ -/*! @page arch-schema-ops Schema Operations - -A @ref arch-schema defines the format of the application data in WiredTiger. -This section details the internals of various schema operations. - -Schema operations cause an update to the metadata and are performed under a -schema lock to avoid concurrent operations on the same object. The following -sequence of steps define a generic schema operation: - -@plantuml_start{schema_generic.png } -@startuml{schema_generic.png} -:A schema operation; -partition with-schema-lock { - :perform operation on underlying data-object; - :update metadata-file; - :checkpoint and sync metadata; -} -stop -@enduml -@plantuml_end - -@section schema_create Schema Create - -The create schema operation is responsible for creating the underlying data -object on the filesystem with the right parameters and then creating an entry -for this new object into the metadata. The sequence of operations involved in a -create for various schema types are as follows: - -@plantuml_start{schema_create.png } -@startuml{schema_create.png} -:WT_SESSION->create(.,name,.) -(__session_create()); - -partition session-API-call { - :API session init with NULL dhandle; - :exit if PANIC flag set; - :exit if invalid configuration; -} - -:validate "name" and if passed "type" config parameter; -note right - "name" parameter is called as "uri" internally. - "type" is usually not passed and generally - implied from the uri. -end note - -partition with-schema-lock { - partition with-table-lock { - :turn on meta tracking; - :check uri} - - split - :uri matches "file:" - ("file" is the underlying - type for all the objects); - split again - :uri matches "colgroup:"; - :__create_colgroup(); - split again - :uri matches "table:"; - :__create_table(); - split again - :uri matches "lsm:"; - :__wt_lsm_tree_create(); - split again - :uri matches "index:"; - :__create_index(); - split again - :matches a named data-source; - :__create_data_source(); - end split - - partition __create_file() { - :exit if file exists; - :validate allocation size; - :block manager creates the file: - 1.create file using __wt_open() - 2.write an initial descriptor to file - 3.fsync and close the file handle; - if (metadata-file?) then (yes) - else (no) - :update metadata with file - configuration and version; - endif - :check if file setup correctly by - getting btree handle with - WT_DHANDLE_EXCLUSIVE set; - if (metatracking on?) then (yes) - :track locked handle*; - else (no) - :release btree - - sync and close; - endif - } - - partition turn-off-meta-tracking { - if (errors?) then (yes) - :unroll operations; - else (no) - if (logging?) then (yes) - :sync log; - else (no) - endif - :checkpoint and sync; - endif - :apply post-commit ops: - release tracked (handle) btree* - - sync and close; - note right - if meta tracking is on, this btree - was being tracked as locked. As part - of tuning off meta tracking, we sync - and close this btree - end note - } - } -} - -:API-end; - -stop -@enduml -@plantuml_end - -@section schema_rename Schema Rename - -The rename schema operation is responsible for renaming the underlying data -object on the filesystem and updating the metadata accordingly. The sequence of -operations involved in a rename for various schema types are as follows: - -@plantuml_start{schema_rename.png } -@startuml{schema_rename.png} -:WT_SESSION->rename(old-uri, new-uri, .) -(__session_rename()); -:session-API-call; - -partition with-checkpoint-lock { - partition with-schema-lock { - partition with-table-write-lock { - :validate new uri-type to match the old type; - :turn on meta tracking; - :check uri} - - split - :uri matches "file:" - ("file" is the underlying - type for all the objects); - split again - :uri matches "lsm:"; - :__wt_lsm_tree_rename(); - split again - :matches a named data-source; - :WT_DATA_SOURCE::rename(); - split again - :uri matches "table:"; - partition __rename_table() { - :rename colgroups and indices represented by the table: - 1. extract names from the uri - 2. create new uri with existing types and configuration - 3. recursive call the rename operation on individual - colgroup and index with the old and the new uri - 4. remove old entry for colgroups and indices from - the metadata table and add the new ones; - :close and remove table handle from the session; - :remove old table entry from the metadata table - and add a new one; - } - end split - - partition __rename_file() { - :fail if backup cursor open and schema operations will conflict; - :close btree handles in the file; - :fail if file with the old name doesn't exist or with the new - name exists; - :remove old file entries and add new in the metadata; - :rename the underlying file; - if (meta-tracking?) then (yes) - :track filesystem op; - else (no) - endif - } - - :bump schema generation number to ignore stale data; - - partition turn-off-meta-tracking { - if (errors?) then (yes) - :unroll operations; - else (no) - if (logging?) then (yes) - :sync log; - else (no) - endif - :checkpoint and sync; - endif - } - } - } -} - -:API-end; - -stop -@enduml -@plantuml_end - -*/ diff --git a/src/third_party/wiredtiger/src/docs/arch-schema.dox b/src/third_party/wiredtiger/src/docs/arch-schema.dox index 6494da597a2..0d6440c7f93 100644 --- a/src/third_party/wiredtiger/src/docs/arch-schema.dox +++ b/src/third_party/wiredtiger/src/docs/arch-schema.dox @@ -1,11 +1,118 @@ /*! @arch_page arch-schema Schema -A schema defines the format of the application data in WiredTiger. WiredTiger -supports various types of schemas (See @ref schema for more -information), operated upon through a WT_SESSION reference. +A schema defines the format of the application data and how it will be stored by WiredTiger. While +many tables have simple key/value pairs for records, WiredTiger also supports more complex data +patterns. See @ref schema for more information. -@subpage arch-schema-ops +@section schema_data_formats Data Formats -This section details the internals of various API operations that operate on schemas, such as WT_SESSION::create and WT_SESSION::rename. +The format of keys and values is configured through \c key_format and \c value_format entries in +@ref config_strings. WiredTiger supports simple or composite data formats for keys and values. +See @ref schema_format_types for the full list of supported data types. + - A simple format stores data in one type, for example \c "key_format=i,value_format=S". + - A composite format can store multiple data types as a single blob. The configuration string + can be something like \c "key_format=Si,value_format=ul". Cursors support encoding and decoding of + these types of keys and values. See @ref arch_cursor_raw and @ref schema_cursor_formats for + more details. +Column store requires the key format to be defined as the record number \c 'r' type. +@ref schema has more information on key/value formats. + +@section schema_files Data Files + +Database schema defines how data files are organized in the database home folder: + - A row-oriented table keeps all the data in one file called \c "<table name>.wt", where + \c "<table name>" is the name that was passed as a part of the \c name parameter to WT_SESSION::create. + - A column-oriented table stores the data in multiple files. One for each column group. The + filename will be as follows \c "<table name>_<colgroup name>.wt". Where \c "<table name>" is + the name that was specified as a part of the \c name parameter to WT_SESSION::create. And + \c "<colgroup name>" is the column group name defined in the \c colgroups entry during the + definition of the table format. See this example of how column groups can be configured in + WiredTiger @ex_ref{ex_col_store.c}. @ref arch-row-column "Row Store and Column Store" describes + in more detail how row and column stores work. + - Each table index is stored in a separate file \c "<table name>_<index name>.wti". + Where \c "<table name>" is the table name passed into WT_SESSION::create. And \c "<index name>" + is the index name defined during index creation. See @ref schema_indices for more + information on how to create a table index. + - LSM trees are store on the file system in \c "<table name>-<chunk id>.lsm" files. Where + \c "<table name>" is the name that was specified as a part of the \c name parameter to + WT_SESSION::create. \c "<chunk id>" is the chunk index managed by WiredTiger. More information + on LSM trees can be found on this page @ref lsm. + - Bloom filters for LSM trees are stored in the files with the same name as the LSM tree chunk, + but with a different extension: "<table name>-<chunk id>.bf". See @ref lsm_bloom for more details. + +@section schema_integrity Schema Integrity + +A user can create and manipulate database objects through the API listed on this page +@ref schema_api. There are several WiredTiger internal objects such as +@ref arch-metadata "Metadata", @ref arch-hs "History Store", etc. The schema of those objects +is locked and cannot be altered from outside of WiredTiger. + +Schema operations cause an update to the metadata and are performed under the schema lock to avoid +concurrent operations on the database schema. The following sequence of steps define a generic schema +operation: + +@plantuml_start{schema_generic.png } +@startuml{schema_generic.png} +:A schema operation; +partition with-schema-lock { + :perform operation on underlying data-object; + :update metadata-file; + :checkpoint and sync metadata; +} +stop +@enduml +@plantuml_end + +Apart from the schema API, the schema lock is necessary for many other +operations in WiredTiger including the following "heavy" database modifications: + - The schema lock wraps `checkpoint prepare` to avoid any tables being created or dropped + during this phase. See @ref arch-checkpoint "Checkpoint" for details. + - `Rollback to stable` operation acquires the schema lock to make sure no schema changes are + done during this complex process. @ref arch-rts "Rollback to Stable" has more information on the + operation. + - A `backup cursor` also holds the schema lock because it must guarantee a consistent view + of what files and tables exist while it is being used, so it prevents any tables or files being + created or dropped during that time. See @ref backup and @ref arch_cursor_backup for + more information. + +All the schema operations listed below perform multi-step metadata modifications. Although they +are \c non-transactional, the schema code tracks the metadata changes and performs the file and +metadata operations in a specific order to provide recovery in the case of a crash. + +@section schema_api Schema Manipulation + +All schema manipulations are done in the context of WT_SESSION. All the methods below, +except WT_SESSION::create and WT_SESSION::truncate, require exclusive access to the specified +data source(s). If any cursors are open with the specified name(s) or a data source is otherwise +in use, the call will fail and return EBUSY. + +@subsection schema_create Create + +The create schema operation is responsible for creating the underlying data +objects on the filesystem and then creating required entries in the metadata. The API for this +operation is WT_SESSION::create. + +@subsection schema_drop Drop + +WT_SESSION::drop operation drops the specified \c uri. The method will delete all related files +and metadata entries. It is possible to keep the underlying files by specifying +\c "remove_files=false" in the config string. + +@subsection schema_rename Rename + +WT_SESSION::rename schema operation renames the underlying data objects on the filesystem and +updates the metadata accordingly. + +@subsection schema_alter Alter + +WT_SESSION::alter allows modification of some table settings after creation. + +@subsection schema_truncate Truncate + +WT_SESSION::truncate truncates a file, table, cursor range, or backup cursor. If \c start and +\c stop cursors are not specified all the data stored in the \c uri will be wiped out. When a range +truncate is in progress, and another transaction inserts a key into that range, the behavior +is not well defined. It is best to avoid this type of situations. See @ref btree_truncate_operation +for more details. */ diff --git a/src/third_party/wiredtiger/src/docs/arch-session.dox b/src/third_party/wiredtiger/src/docs/arch-session.dox index e17eca10b1b..fda4f3e339f 100644 --- a/src/third_party/wiredtiger/src/docs/arch-session.dox +++ b/src/third_party/wiredtiger/src/docs/arch-session.dox @@ -1,7 +1,8 @@ /*! @arch_page arch-session Session @section arch_session_def Definition -After a @ref arch-connection has been established between the application and WiredTiger, the +After a @ref arch-connection "Connection" has been established between the application and +WiredTiger, the application can start sending requests to WiredTiger using a session. A session is internally represented by WT_SESSION and plays an important role since almost all operations are performed under the context of a session. @@ -13,7 +14,7 @@ associated with one connection. The maximum number of sessions is set through th \c session_max as part of the configuration string in ::wiredtiger_open. Sessions created by the calling application are called "user sessions". WiredTiger also performs -some internal operations such as @ref arch-eviction through self-created sessions. These sessions +some internal operations such as @ref arch-eviction "Eviction" through self-created sessions. These sessions are called "internal sessions". The usage rules and guidelines for both internal sessions and user sessions are the same and the only difference between them is their origin of creation. @@ -24,23 +25,24 @@ documentation related to WT_SESSION. @section arch_session_txn Transactions It is possible to group several operations within a session, in other words, multiple operations can -be treated as a single atomic operation. This can be done using @ref arch-transaction. Furthermore, +be treated as a single atomic operation. This can be done using @ref arch-transaction +"Transactions". Furthermore, a session can hold only one running transaction at any given time and this transaction only belongs to that session. @section arch_session_cur Cursors A session can perform multiple data operations on one or several collections using multiple cursors -(see @ref arch-cursor for more details). All the cursors associated with a session share that +(see @ref arch-cursor "Cursor" for more details). All the cursors associated with a session share that session transaction context. It is also possible to cache those cursors if required through the configuration string given to WT_CONNECTION::open_session or ::wiredtiger_open. The configuration item for this purpose is \c cache_cursors. @section arch_session_dhandles Data Handles -During its lifetime, a session can accumulate a list of data handles (see @ref arch-dhandle). +During its lifetime, a session can accumulate a list of data handles (see @ref arch-dhandle "Data Handles"). Indeed, when a session accesses a table for the first time, the data handle of that table is acquired and cached. Once a session no longer needs to operate on a table, it marks the associated data handle as idle. This helps the sweep server release data handles that are inactive, see @ref -arch-dhandle-lifecycle for more details. +arch-dhandle for more details. @section arch_session_closure Closure A session can be closed using WT_SESSION::close. Closing the connection will also close all opened diff --git a/src/third_party/wiredtiger/src/docs/arch-snapshot.dox b/src/third_party/wiredtiger/src/docs/arch-snapshot.dox index 8f8142d1f33..4f9618f56b1 100644 --- a/src/third_party/wiredtiger/src/docs/arch-snapshot.dox +++ b/src/third_party/wiredtiger/src/docs/arch-snapshot.dox @@ -1,9 +1,170 @@ /*! @arch_page arch-snapshot Snapshot -When the snapshot isolation level is used in WiredTiger, the transaction -will see versions of records committed before the transaction started. +Transaction snapshots are an important mechanism for providing strong isolation guarantees +in WiredTiger. WiredTiger will often maintain multiple versions of the users data throughout +its lifetime, with which snapshots provide the mechanisms to ensure particular versions of that data are +made accessible. Snapshots are importantly leveraged by WiredTiger to define the visibility rules for data such +as isolating concurrent transactions from reading each others updates. + +@section snapshot_transaction Transactions using Snapshots + +Within the context of a WiredTiger transaction, a snapshot ensures specific versions of a record are accessible based on the +time the snapshot was created. In particular, these accessibility rules are defined by a transactions isolation level. +Transactions more specifically use snapshots for the following isolation levels: + +- \c snapshot: A snapshot is created at the beginning of a snapshot-level isolated transaction and is sustained throughout +the entire life cycle of the transaction. In effect, a transaction using \c snapshot isolation will only read versions of +records before the transaction started. The default isolation level is \c snapshot. +- \c read-committed: Read-committed transactions ensure a transaction is able to see committed updates by other transactions +at the time of reading. To achieve this, under a read-committed isolation, a snapshot is created when performing a search. The +same snapshot is used for all subsequent reads until another search is called. + +See @ref transactions for further details about how WiredTiger transactions utilize snapshots to create isolation guarantees and +implement concurrency control. + +@section snapshot_internals Snapshot Internals + +Snapshots are implemented by capturing the global state of transactions in the system at the time of snapshot creation. Snapshots +are managed within per-session transaction contexts, encapsulated by the \c WT_TXN data structure. The main +data points that encapsulate a given snapshot include: + +- Maximum transaction ID: The maximum global transaction ID value seen when creating the snapshot. This being the upper +bound value which all concurrent transactions IDs are currently smaller than. We resolve the maximum transaction ID value by reading the +global current transaction ID/counter at the time of snapshot creation. Transactions with IDs greater than or equal to the maximum ID are +considered invisible to the snapshot. +- Concurrent transaction IDs: A list of transaction IDs that are concurrently active at the time of the snapshot creation. As these +transactions are yet to be committed at the time of snapshot creation, transactions stored in this list are considered invisible when performing visibility checks. +When building the list of concurrent transactions, WiredTiger walks a global array of running transactions, storing each walked transaction ID in a list managed by the snapshot. +When walking the global set of running transactions, WiredTiger will ignore a transaction ID that is: + -# Its own ID ie. the ID of the transaction creating the given snapshot. A transaction will always read its own updates. + -# An ID that is greater than or equal to the snapshot's maximum transaction ID. As WiredTiger walks the global list, + it may find a transaction that has concurrently allocated a new transaction ID, exceeding the snapshots maximum ID bounds. This will + however be ignored as it is not considered visible at the time of snapshot creation. +- Minimum transaction ID: The minimum global transaction ID value seen when creating the snapshot. This being the lower +bound value which all concurrent transactions IDs are greater than or equal to. Transaction IDs strictly less than the minimum value +are considered visible as these would've been allocated and committed before the snapshot was created. This value is usually taken to be the +smallest transaction ID value found from the list of concurrent transactions. + +An example of building out the aforementioned components of a snapshot can be seen in the below figure. This figure illustrates constructing a snapshot for a transaction with +ID \c 8. Consider that at the given time, the concurrent running transactions are \c 3, \c 5, \c 7, \c 8 and the global current transaction ID is \c 9. + +@plantuml_start{snapshot.png} +@startuml{snapshot.png} +digraph g { +graph [ +ordering="in"; +rankdir = "LR"; +]; +node [ +fontsize = "16" +]; +edge [ +]; +"node1" [ +label = "<f0> Snapshot [TXID 8] | <f1> Concurrent Transactions | <f2> Maximum Transaction ID | <f3> Minimum Transaction ID" +shape = "record" +]; +"node0" [ +label = "<f0> Global Transaction State | <f1> Running Transactions| <f2> Current Transaction ID" +shape = "record" +]; +"node2" [ +label = "<f0> 3" +shape = "record" +]; +"node3" [ +label = "<f0> 3" +shape = "record" +]; +"node4" [ +label = "<f0> 5" +shape = "record" +]; +"node6" [ +label = "<f0> 7" +shape = "record" +]; +"node7" [ +label = "<f0> 8" +shape = "record" +]; +"node8" [ +label = "<f0> 5" +shape = "record" +]; +"node10" [ +label = "<f0> 7" +shape = "record" +]; +"node11" [ +label = "<f0> 9" +shape = "record" +]; +"node12" [ +label = "<f0> 3" +shape = "record" +]; +"node13" [ +label = "<f0> 9" +shape = "record" +]; +"node1":f1 -> "node2":f0 [ +id = 0 +]; +"node2":f0 -> "node8":f0 [ +id = 0 +]; +"node8":f0 -> "node10":f0 [ +id = 0 +]; +"node0":f1 -> "node3":f0 [ +id = 0 +]; +"node3":f0 -> "node4":f0 [ +id = 0 +]; +"node4":f0 -> "node6":f0 [ +id = 0 +]; +"node6":f0 -> "node7":f0 [ +id = 0 +]; +"node0":f2 -> "node11":f0 [ +id = 0 +]; +"node1":f2 -> "node13":f0 [ +id = 0 +]; +"node1":f3 -> "node12":f0 [ +id = 0 +]; +} +@enduml +@plantuml_end + +@section snapshot_visibility Snapshot Visibility + +When determining if a transaction can read a given record, WiredTiger applies a visibility function to discern whether the committed value is +visible. When using \c snapshot and \c read-committed isolated transactions, WiredTiger's visibility function will leverage a transactions snapshot information +to determine if a given record is visible. When determining if a given transaction ID is visible to a snapshot, +WiredTiger applies the below checks in the following order: +1. The transaction ID is greater than or equal to the snapshots recorded maximum transaction ID: If satisfied, the transaction ID is invisible, as the ID +would've been allocated after the snapshot was created. +2. The transaction ID is less than the snapshots recorded minimum transaction ID: If satisfied, the transaction ID is visible, as these would've been +allocated and committed before the snapshot was created. +3. The transaction ID exists in the snapshots recorded concurrent transaction list: If found, the transaction ID is invisible, as this ID wasn't committed at the +time the snapshot was created. +4. Otherwise the ID is visible. + +The early ordering of checking whether the transaction ID sits outside the maximum and minimum bounds is an important detail since the concurrent transaction list +can be empty i.e. snapshot was taken at a time when no concurrent transactions were running. + +@section snapshot_checkpoint Snapshots \& Checkpoints + +Checkpoints require a snapshot to write a transactionally consistent view of the data. A checkpoint maintains its own special global +transaction and snapshot state. This being independent from the application's running transactions. + +When checkpoints are running, application transactions using snapshots will also store the checkpoints transaction ID as a concurrent transaction. +This allowing a snapshot to ignore any uncommitted changes the checkpoint has written to the metadata. -Snapshots are implemented by storing the set of transaction ids committed -before the transaction started in the \c WT_TXN structure. Transaction -ids older than a certain point need not be stored. */ diff --git a/src/third_party/wiredtiger/src/docs/arch-transaction.dox b/src/third_party/wiredtiger/src/docs/arch-transaction.dox index c1be0f99394..c9272356717 100644 --- a/src/third_party/wiredtiger/src/docs/arch-transaction.dox +++ b/src/third_party/wiredtiger/src/docs/arch-transaction.dox @@ -159,7 +159,7 @@ store to check if there is a version visible to the reader there. @subsection Durability WiredTiger transactions support commit level durability and checkpoint level durability. An -operation is commit level durable if logging is enabled on the table (@ref arch-logging). After it +operation is commit level durable if @ref arch-logging "Logging" is enabled on the table. After it has been successfully committed, the operation is guaranteed to survive restart. An operation will only survive across restart under checkpoint durability if it is included in the last successful checkpoint. @@ -182,8 +182,8 @@ timestamp, which may cause excessive amounts of data to be pinned in memory. The is allowed to move beyond the prepared timestamp and at the commit time, the prepared transaction can then be committed after the current stable timestamp with a larger durable timestamp. The durable timestamp also marks the time the update is to be stable. If the stable timestamp is moved -to or beyond the durable timestamp of an update, it will not be removed by rollback to stable from -a checkpoint. See @ref arch-rts for more details. +to or beyond the durable timestamp of an update, it will not be removed from a checkpoint by +@ref arch-rts "rollback to stable". The visibility of the prepared transaction is also special when in the prepared state. Since in the prepared state, the transaction has released its snapshot, it should be visible to the transactions diff --git a/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/snapshot.png b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/snapshot.png Binary files differnew file mode 100644 index 00000000000..ac2e237074d --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/images/plantuml_gen_img/snapshot.png diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 182a9af00f3..f34eace96d9 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -51,6 +51,7 @@ ENOTSUP EXTLIST EmpId Encryptors +FSTREAM Facebook Facebook's FlameGraph @@ -114,7 +115,6 @@ RLE RedHat RepMgr Riak -RLE RocksDB Roelofs RrVv @@ -125,6 +125,7 @@ Sanitizer Seward's SiH TSC +TXID TXT UBSAN UNC @@ -151,6 +152,7 @@ Yann Za Zstd aR +aarch abc abstime ack'ed @@ -177,17 +179,18 @@ benchmarking bigram bindir bitstring +bm bokeh bool boolean booleans -bm br bt +btcur btmem btree -btrees btree's +btrees bufs builtin builtins @@ -288,8 +291,8 @@ dup dups encrypt encryptor -encryptors encryptor's +encryptors endcode endcond endian @@ -302,6 +305,7 @@ eof eq erlang errno +evictable exe extensibility fadvise @@ -329,6 +333,7 @@ flamegraph flamegraphs flcs fnv +fontsize forw fprofile fput @@ -337,6 +342,7 @@ fs fsanitize fsync ftruncate +func fuzzer fuzzers fuzzutil @@ -370,6 +376,7 @@ iflag incr indices init +inmemory insn intl intpack @@ -441,6 +448,7 @@ memp metadata metatracking minkey +mips mispredictions mixin mixins @@ -509,6 +517,7 @@ pid plantuml png posix +ppc pre prealloc prepended @@ -533,6 +542,7 @@ py qnx qqq rVv +rankdir rdbms rdlock readlock @@ -581,6 +591,7 @@ skinparam skiplist skiplists sortable +sparc spinlock spinlocks sql @@ -671,6 +682,7 @@ workgen writelock writelocks wrlock +wti wtperf wtstats xa diff --git a/src/third_party/wiredtiger/src/docs/tool-optrack.dox b/src/third_party/wiredtiger/src/docs/tool-optrack.dox index e39a17e3a08..0d1c2d3b110 100644 --- a/src/third_party/wiredtiger/src/docs/tool-optrack.dox +++ b/src/third_party/wiredtiger/src/docs/tool-optrack.dox @@ -76,7 +76,7 @@ connection configuration string. For example, if you are running `wtperf`, you would add it to the `conn_config` string: - ./wtperf -c conn_config="operation_tracking=(enabled=true,path=.)" + ./wtperf -C conn_config="operation_tracking=(enabled=true,path=.)" The "path" argument determines where the log files, produced by operation tracking, will be stored. Now you can run your workload. diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index b0cada52261..0488f2346f6 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -1522,6 +1522,16 @@ retry: } __wt_spin_unlock(session, &cache->evict_walk_lock); WT_ERR(ret); + /* + * If there is a checkpoint thread gathering handles, which means it is holding the + * schema lock, then there is often contention on the evict walk lock with that thread. + * If eviction is not in aggressive mode, sleep a bit to give the checkpoint thread a + * chance to gather its handles. + */ + if (F_ISSET(conn, WT_CONN_CKPT_GATHER) && !__wt_cache_aggressive(session)) { + __wt_sleep(0, 10); + WT_STAT_CONN_INCR(session, cache_eviction_walk_sleeps); + } } } diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index de249ec4a4b..eab73e9dc07 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -95,6 +95,7 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32 WT_DECL_RET; WT_PAGE *page; uint64_t time_start, time_stop; + uint32_t i; bool clean_page, closing, force_evict_hs, inmem_split, local_gen, tree_dead; conn = S2C(session); @@ -222,6 +223,25 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32 if (0) { err: + /* + * Mark the page dirty again if it has any prepared updates that are yet to be resolved + * after a successful reconciliation and the page is failed to evict. Otherwise, the + * resolved prepared updates are lost as the eviction don't reconcile page again and uses + * the older reconciled image blocks to proceed with the eviction. + */ + if (ref->page != NULL && ref->page->modify != NULL && + ref->page->modify->page_state == WT_PAGE_CLEAN) { + if (ref->page->modify->rec_result == WT_PM_REC_REPLACE && + ref->page->modify->u1.r.replace.ta.prepare) + __wt_page_modify_set(session, ref->page); + else if (ref->page->modify->rec_result == WT_PM_REC_MULTIBLOCK) { + for (i = 0; i < ref->page->modify->u1.m.multi_entries; i++) { + if (ref->page->modify->u1.m.multi[i].addr.ta.prepare) + __wt_page_modify_set(session, ref->page); + } + } + } + if (!closing) __evict_exclusive_clear(session, ref, previous_state); diff --git a/src/third_party/wiredtiger/src/history/hs_rec.c b/src/third_party/wiredtiger/src/history/hs_rec.c index 777c2a74863..2da95291c12 100644 --- a/src/third_party/wiredtiger/src/history/hs_rec.c +++ b/src/third_party/wiredtiger/src/history/hs_rec.c @@ -356,8 +356,8 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_MULTI *mult break; case WT_PAGE_ROW_LEAF: if (list->ins == NULL) { - WT_WITH_BTREE(session, btree, - ret = __wt_row_leaf_key(session, r->page, list->ripcip, key, false)); + WT_WITH_BTREE( + session, btree, ret = __wt_row_leaf_key(session, r->page, list->rip, key, false)); WT_ERR(ret); } else { key->data = WT_INSERT_KEY(list->ins); diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 20a86779a91..6f9c350afd6 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -14,14 +14,16 @@ * is safe to use the reference count without atomic operations because the reference count is only * tracking a thread re-entering the API. */ -#define WT_SINGLE_THREAD_CHECK_START(s) \ - { \ - uintmax_t __tmp_api_tid; \ - __wt_thread_id(&__tmp_api_tid); \ - WT_ASSERT(session, (s)->id == 0 || (s)->api_tid == 0 || (s)->api_tid == __tmp_api_tid); \ - if ((s)->api_tid == 0) \ - WT_PUBLISH((s)->api_tid, __tmp_api_tid); \ - ++(s)->api_enter_refcnt; \ +#define WT_SINGLE_THREAD_CHECK_START(s) \ + { \ + uintmax_t __tmp_api_tid; \ + __wt_thread_id(&__tmp_api_tid); \ + WT_ASSERT(s, (s)->id == 0 || (s)->api_tid == 0 || (s)->api_tid == __tmp_api_tid); \ + if ((s)->api_tid == 0) { \ + (s)->api_tid = __tmp_api_tid; \ + WT_FULL_BARRIER(); \ + } \ + ++(s)->api_enter_refcnt; \ } #define WT_SINGLE_THREAD_CHECK_STOP(s) \ @@ -36,7 +38,7 @@ WT_DATA_HANDLE *__olddh = (s)->dhandle; \ const char *__oldname; \ /* If this isn't an API reentry, the name should be NULL and the counter should be 0. */ \ - WT_ASSERT(session, (s)->name != NULL || (s)->api_call_counter == 0); \ + WT_ASSERT(s, (s)->name != NULL || (s)->api_call_counter == 0); \ __oldname = (s)->name; \ ++(s)->api_call_counter; \ (s)->dhandle = (dh); \ @@ -73,7 +75,7 @@ const char *(cfg)[] = {WT_CONFIG_BASE(s, h##_##n), config, NULL}; \ API_SESSION_INIT(s, h, n, dh); \ if ((config) != NULL) \ - WT_ERR(__wt_config_check((s), WT_CONFIG_REF(session, h##_##n), (config), 0)) + WT_ERR(__wt_config_check((s), WT_CONFIG_REF(s, h##_##n), (config), 0)) #define API_END(s, ret) \ if ((s) != NULL) { \ @@ -81,7 +83,7 @@ WT_SINGLE_THREAD_CHECK_STOP(s); \ if ((ret) != 0) \ __wt_txn_err_set(s, ret); \ - if ((s)->api_call_counter == 1 && !F_ISSET(session, WT_SESSION_INTERNAL)) \ + if ((s)->api_call_counter == 1 && !F_ISSET(s, WT_SESSION_INTERNAL)) \ __wt_op_timer_stop(s); \ /* \ * We should not leave any history store cursor open when return from an api call. \ @@ -212,14 +214,14 @@ TXN_API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define CURSOR_API_CALL(cur, s, n, bt) \ - (s) = (WT_SESSION_IMPL *)(cur)->session; \ + (s) = CUR2S(cur); \ SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ API_CALL_NOCONF(s, WT_CURSOR, n, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ if (F_ISSET(cur, WT_CURSTD_CACHED)) \ WT_ERR(__wt_cursor_cached(cur)) #define CURSOR_API_CALL_PREPARE_ALLOWED(cur, s, n, bt) \ - (s) = (WT_SESSION_IMPL *)(cur)->session; \ + (s) = CUR2S(cur); \ API_CALL_NOCONF(s, WT_CURSOR, n, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ if (F_ISSET(cur, WT_CURSTD_CACHED)) \ WT_ERR(__wt_cursor_cached(cur)) @@ -237,7 +239,7 @@ JOINABLE_CURSOR_CALL_CHECK(cur) #define CURSOR_REMOVE_API_CALL(cur, s, bt) \ - (s) = (WT_SESSION_IMPL *)(cur)->session; \ + (s) = CUR2S(cur); \ SESSION_API_PREPARE_CHECK(s, WT_CURSOR, remove); \ TXN_API_CALL_NOCONF(s, WT_CURSOR, remove, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle) @@ -246,7 +248,7 @@ JOINABLE_CURSOR_CALL_CHECK(cur) #define CURSOR_UPDATE_API_CALL_BTREE(cur, s, n) \ - (s) = (WT_SESSION_IMPL *)(cur)->session; \ + (s) = CUR2S(cur); \ SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ TXN_API_CALL_NOCONF(s, WT_CURSOR, n, ((WT_CURSOR_BTREE *)(cur))->dhandle); \ if (F_ISSET(S2C(s), WT_CONN_IN_MEMORY) && !F_ISSET(CUR2BT(cur), WT_BTREE_IGNORE_CACHE) && \ @@ -254,7 +256,7 @@ WT_ERR(WT_CACHE_FULL); #define CURSOR_UPDATE_API_CALL(cur, s, n) \ - (s) = (WT_SESSION_IMPL *)(cur)->session; \ + (s) = CUR2S(cur); \ SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ TXN_API_CALL_NOCONF(s, WT_CURSOR, n, NULL) diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 211f2f220b0..6382a1d5f59 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -253,7 +253,7 @@ struct __wt_ovfl_reuse { */ struct __wt_save_upd { WT_INSERT *ins; /* Insert list reference */ - WT_ROW *ripcip; /* Original on-page reference */ + WT_ROW *rip; /* Original on-page reference */ WT_UPDATE *onpage_upd; bool restore; /* Whether to restore this saved update chain */ }; @@ -1026,7 +1026,7 @@ struct __wt_row { /* On-page key, on-page cell, or off-page WT_IKEY */ * WT_ROW_SLOT -- * Return the 0-based array offset based on a WT_ROW reference. */ -#define WT_ROW_SLOT(page, rip) ((uint32_t)(((WT_ROW *)(rip)) - (page)->pg_row)) +#define WT_ROW_SLOT(page, rip) ((uint32_t)((rip) - (page)->pg_row)) /* * WT_COL -- Each in-memory variable-length column-store leaf page has an array of WT_COL @@ -1065,7 +1065,7 @@ struct __wt_col { * WT_COL_SLOT -- * Return the 0-based array offset based on a WT_COL reference. */ -#define WT_COL_SLOT(page, cip) ((uint32_t)(((WT_COL *)(cip)) - (page)->pg_var)) +#define WT_COL_SLOT(page, cip) ((uint32_t)((cip) - (page)->pg_var)) /* * WT_IKEY -- diff --git a/src/third_party/wiredtiger/src/include/btree_cmp_inline.h b/src/third_party/wiredtiger/src/include/btree_cmp_inline.h index 0c7eaf9fdb9..ffa9d51a9f1 100644 --- a/src/third_party/wiredtiger/src/include/btree_cmp_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_cmp_inline.h @@ -21,8 +21,8 @@ /* * __wt_lex_compare -- - * Lexicographic comparison routine. Returns: < 0 if user_item is lexicographically < tree_item - * = 0 if user_item is lexicographically = tree_item > 0 if user_item is lexicographically > + * Lexicographic comparison routine. Returns: < 0 if user_item is lexicographically < tree_item, + * = 0 if user_item is lexicographically = tree_item, > 0 if user_item is lexicographically > * tree_item. We use the names "user" and "tree" so it's clear in the btree code which the * application is looking at when we call its comparison function. If prefix is specified, 0 can * be returned when the user_item is equal to the tree_item for the minimum size. @@ -88,12 +88,28 @@ __wt_lex_compare(const WT_ITEM *user_item, const WT_ITEM *tree_item, bool prefix /* * Use the non-vectorized version for the remaining bytes and for the small key sizes. */ - for (; len > 0; --len, ++userp, ++treep) + for (; len > 0; --len, ++userp, ++treep) { + /* + * When prefix is enabled and we are performing lexicographic comparison on schema formats s + * or S, we only want to compare the characters before either of them reach a NUL character. + * For format S, a NUL character is always at the end of the string, while for the format s, + * NUL characters are set for the remaining unused bytes. If we are at the end of the user + * item (which is the prefix here), there is a prefix match. Otherwise, the tree item is + * lexicographically smaller than the prefix. + */ + if (prefix && (*userp == '\0' || *treep == '\0')) + return (*userp == '\0' ? 0 : 1); if (*userp != *treep) return (*userp < *treep ? -1 : 1); + } - /* Contents are equal up to the smallest length. */ - return ((usz == tsz || prefix) ? 0 : (usz < tsz) ? -1 : 1); + /* + * Contents are equal up to the smallest length. In the case of a prefix match, we consider the + * tree item and the prefix equal only if the tree item is bigger in size. + */ + if (usz == tsz || (prefix && usz < tsz)) + return (0); + return ((usz < tsz) ? -1 : 1); } /* diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index e40bc2acc27..c94dd274fbc 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -38,6 +38,7 @@ extern WT_PROCESS __wt_process; struct __wt_bucket_storage { const char *bucket; /* Bucket name */ const char *bucket_prefix; /* Bucket prefix */ + const char *cache_directory; /* Locally cached file location */ int owned; /* Storage needs to be terminated */ uint64_t object_size; /* Tiered object size */ uint64_t retain_secs; /* Tiered period */ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index de46a4347be..3468dbfa24b 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1243,7 +1243,7 @@ extern int __wt_rec_split_finish(WT_SESSION_IMPL *session, WT_RECONCILE *r) extern int __wt_rec_split_init(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page, uint64_t recno, uint64_t max) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, - void *ripcip, WT_CELL_UNPACK_KV *vpack, WT_UPDATE_SELECT *upd_select) + WT_ROW *rip, WT_CELL_UNPACK_KV *vpack, WT_UPDATE_SELECT *upd_select) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -1363,6 +1363,8 @@ extern int __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM ** WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_blocking_checkpoint(WT_SESSION_IMPL *session, bool force, uint64_t seconds) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_breakpoint(WT_SESSION *wt_session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_close_internal(WT_SESSION_IMPL *session) @@ -1810,7 +1812,7 @@ extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_random_init_seed(WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_read_cell_time_window(WT_CURSOR_BTREE *cbt, WT_REF *ref, WT_TIME_WINDOW *tw); +extern void __wt_read_cell_time_window(WT_CURSOR_BTREE *cbt, WT_TIME_WINDOW *tw, bool *tw_foundp); extern void __wt_read_row_time_window( WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, WT_TIME_WINDOW *tw); extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index f103e4f56c8..d52956ee3d2 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -46,6 +46,9 @@ #define WT_SYSTEM_BASE_WRITE_GEN_URI "system:checkpoint_base_write_gen" /* Base write gen URI */ #define WT_SYSTEM_BASE_WRITE_GEN "base_write_gen" /* Base write gen name */ +/* Check whether a string is a legal URI for a btree object */ +#define WT_BTREE_PREFIX(str) (WT_PREFIX_MATCH(str, "file:") || WT_PREFIX_MATCH(str, "tiered:")) + /* * Optimize comparisons against the metafile URI, flag handles that reference the metadata file. */ diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index fa97bd881c9..2eb26c749dc 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -369,6 +369,7 @@ struct __wt_connection_stats { int64_t cache_eviction_slow; int64_t cache_eviction_walk_leaf_notfound; int64_t cache_eviction_state; + int64_t cache_eviction_walk_sleeps; int64_t cache_eviction_target_page_lt10; int64_t cache_eviction_target_page_lt32; int64_t cache_eviction_target_page_ge128; @@ -693,6 +694,7 @@ struct __wt_connection_stats { int64_t session_query_ts; int64_t session_table_alter_fail; int64_t session_table_alter_success; + int64_t session_table_alter_trigger_checkpoint; int64_t session_table_alter_skip; int64_t session_table_compact_fail; int64_t session_table_compact_success; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 5dce2471949..cb433664094 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -317,30 +317,31 @@ struct __wt_txn { */ /* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_TXN_AUTOCOMMIT 0x000001u -#define WT_TXN_ERROR 0x000002u -#define WT_TXN_HAS_ID 0x000004u -#define WT_TXN_HAS_SNAPSHOT 0x000008u -#define WT_TXN_HAS_TS_COMMIT 0x000010u -#define WT_TXN_HAS_TS_DURABLE 0x000020u -#define WT_TXN_HAS_TS_PREPARE 0x000040u -#define WT_TXN_IGNORE_PREPARE 0x000080u -#define WT_TXN_PREPARE 0x000100u -#define WT_TXN_READONLY 0x000200u -#define WT_TXN_RUNNING 0x000400u -#define WT_TXN_SHARED_TS_DURABLE 0x000800u -#define WT_TXN_SHARED_TS_READ 0x001000u -#define WT_TXN_SYNC_SET 0x002000u -#define WT_TXN_TS_READ_BEFORE_OLDEST 0x004000u -#define WT_TXN_TS_ROUND_PREPARED 0x008000u -#define WT_TXN_TS_ROUND_READ 0x010000u -#define WT_TXN_TS_WRITE_ALWAYS 0x020000u -#define WT_TXN_TS_WRITE_KEY_CONSISTENT 0x040000u -#define WT_TXN_TS_WRITE_MIXED_MODE 0x080000u -#define WT_TXN_TS_WRITE_NEVER 0x100000u -#define WT_TXN_TS_WRITE_ORDERED 0x200000u -#define WT_TXN_UPDATE 0x400000u -#define WT_TXN_VERB_TS_WRITE 0x800000u +#define WT_TXN_AUTOCOMMIT 0x0000001u +#define WT_TXN_ERROR 0x0000002u +#define WT_TXN_HAS_ID 0x0000004u +#define WT_TXN_HAS_SNAPSHOT 0x0000008u +#define WT_TXN_HAS_TS_COMMIT 0x0000010u +#define WT_TXN_HAS_TS_DURABLE 0x0000020u +#define WT_TXN_HAS_TS_PREPARE 0x0000040u +#define WT_TXN_IGNORE_PREPARE 0x0000080u +#define WT_TXN_PREPARE 0x0000100u +#define WT_TXN_PREPARE_IGNORE_API_CHECK 0x0000200u +#define WT_TXN_READONLY 0x0000400u +#define WT_TXN_RUNNING 0x0000800u +#define WT_TXN_SHARED_TS_DURABLE 0x0001000u +#define WT_TXN_SHARED_TS_READ 0x0002000u +#define WT_TXN_SYNC_SET 0x0004000u +#define WT_TXN_TS_READ_BEFORE_OLDEST 0x0008000u +#define WT_TXN_TS_ROUND_PREPARED 0x0010000u +#define WT_TXN_TS_ROUND_READ 0x0020000u +#define WT_TXN_TS_WRITE_ALWAYS 0x0040000u +#define WT_TXN_TS_WRITE_KEY_CONSISTENT 0x0080000u +#define WT_TXN_TS_WRITE_MIXED_MODE 0x0100000u +#define WT_TXN_TS_WRITE_NEVER 0x0200000u +#define WT_TXN_TS_WRITE_ORDERED 0x0400000u +#define WT_TXN_UPDATE 0x0800000u +#define WT_TXN_VERB_TS_WRITE 0x1000000u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index b57f0d8203f..908fe1d38dd 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -13,6 +13,8 @@ static inline int __wt_txn_context_prepare_check(WT_SESSION_IMPL *session) { + if (F_ISSET(session->txn, WT_TXN_PREPARE_IGNORE_API_CHECK)) + return (0); if (F_ISSET(session->txn, WT_TXN_PREPARE)) WT_RET_MSG(session, EINVAL, "not permitted in a prepared transaction"); return (0); @@ -1310,14 +1312,14 @@ __wt_txn_modify_check( WT_TIME_WINDOW tw; WT_TXN *txn; WT_TXN_GLOBAL *txn_global; - bool ignore_prepare_set, rollback; + bool ignore_prepare_set, rollback, tw_found; rollback = false; txn = session->txn; txn_global = &S2C(session)->txn_global; /* Don't check if transaction isolation is not snapshot or the table is metadata. */ - if (txn->isolation != WT_ISO_SNAPSHOT || (cbt != NULL && WT_IS_METADATA(cbt->dhandle))) + if (txn->isolation != WT_ISO_SNAPSHOT || WT_IS_METADATA(cbt->dhandle)) return (0); if (txn_global->debug_rollback != 0 && @@ -1339,17 +1341,18 @@ __wt_txn_modify_check( WT_ASSERT(session, upd != NULL || !rollback); /* - * Check conflict against the on page value if there is no update on the update chain except + * Check conflict against any on-page value if there is no update on the update chain except * aborted updates. Otherwise, we would have either already detected a conflict if we saw an * uncommitted update or determined that it would be safe to write if we saw a committed update. */ - if (!rollback && upd == NULL && cbt != NULL && CUR2BT(cbt)->type != BTREE_COL_FIX && - cbt->ins == NULL) { - __wt_read_cell_time_window(cbt, cbt->ref, &tw); - if (WT_TIME_WINDOW_HAS_STOP(&tw)) - rollback = !__wt_txn_tw_stop_visible(session, &tw); - else - rollback = !__wt_txn_tw_start_visible(session, &tw); + if (!rollback && upd == NULL) { + __wt_read_cell_time_window(cbt, &tw, &tw_found); + if (tw_found) { + if (WT_TIME_WINDOW_HAS_STOP(&tw)) + rollback = !__wt_txn_tw_stop_visible(session, &tw); + else + rollback = !__wt_txn_tw_start_visible(session, &tw); + } } if (rollback) { diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index f35b643db2b..daba823f589 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1281,11 +1281,15 @@ struct __wt_session { * table., a string; default empty.} * @config{ bucket_prefix, the * unique bucket prefix for this table., a string; default empty.} - * @config{ local_retention, time in seconds to retain data on tiered - * storage on the local tier for faster read access., an integer between 0 and 10000; - * default \c 300.} - * @config{ name, permitted values are \c "none" or - * custom storage source name created with WT_CONNECTION::add_storage_source. See @ref + * @config{ cache_directory, a directory to store locally cached + * versions of files in the storage source. By default\, it is named with \c "-cache" + * appended to the bucket name. A relative directory name is relative to the home + * directory., a string; default empty.} + * @config{ local_retention, + * time in seconds to retain data on tiered storage on the local tier for faster read + * access., an integer between 0 and 10000; default \c 300.} + * @config{ name, permitted values are \c "none" or custom storage + * source name created with WT_CONNECTION::add_storage_source. See @ref * custom_storage_sources for more information., a string; default \c none.} * @config{ object_target_size, the approximate size of objects * before creating them on the tiered storage tier., an integer between 100K and 10TB; @@ -5119,1066 +5123,1073 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_WALK_LEAF_NOTFOUND 1054 /*! cache: eviction state */ #define WT_STAT_CONN_CACHE_EVICTION_STATE 1055 +/*! + * cache: eviction walk most recent sleeps for checkpoint handle + * gathering + */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SLEEPS 1056 /*! cache: eviction walk target pages histogram - 0-9 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1056 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1057 /*! cache: eviction walk target pages histogram - 10-31 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1057 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1058 /*! cache: eviction walk target pages histogram - 128 and higher */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1058 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1059 /*! cache: eviction walk target pages histogram - 32-63 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1059 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1060 /*! cache: eviction walk target pages histogram - 64-128 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1060 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1061 /*! * cache: eviction walk target pages reduced due to history store cache * pressure */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_REDUCED 1061 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_REDUCED 1062 /*! cache: eviction walk target strategy both clean and dirty pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_BOTH_CLEAN_AND_DIRTY 1062 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_BOTH_CLEAN_AND_DIRTY 1063 /*! cache: eviction walk target strategy only clean pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_CLEAN 1063 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_CLEAN 1064 /*! cache: eviction walk target strategy only dirty pages */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_DIRTY 1064 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_STRATEGY_DIRTY 1065 /*! cache: eviction walks abandoned */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1065 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1066 /*! cache: eviction walks gave up because they restarted their walk twice */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1066 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1067 /*! * cache: eviction walks gave up because they saw too many pages and * found no candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1067 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1068 /*! * cache: eviction walks gave up because they saw too many pages and * found too few candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1068 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1069 /*! cache: eviction walks reached end of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1069 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1070 /*! cache: eviction walks restarted */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_RESTART 1070 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_RESTART 1071 /*! cache: eviction walks started from root of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1071 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1072 /*! cache: eviction walks started from saved location in tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1072 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1073 /*! cache: eviction worker thread active */ -#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1073 +#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1074 /*! cache: eviction worker thread created */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1074 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1075 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1075 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1076 /*! cache: eviction worker thread removed */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1076 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1077 /*! cache: eviction worker thread stable number */ -#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1077 +#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1078 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1078 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1079 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1079 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1080 /*! cache: force re-tuning of eviction workers once in a while */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1080 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1081 /*! * cache: forced eviction - history store pages failed to evict while * session has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_FAIL 1081 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_FAIL 1082 /*! * cache: forced eviction - history store pages selected while session * has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS 1082 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS 1083 /*! * cache: forced eviction - history store pages successfully evicted * while session has history store cursor open */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_SUCCESS 1083 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_HS_SUCCESS 1084 /*! cache: forced eviction - pages evicted that were clean count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN 1084 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN 1085 /*! cache: forced eviction - pages evicted that were clean time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN_TIME 1085 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_CLEAN_TIME 1086 /*! cache: forced eviction - pages evicted that were dirty count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY 1086 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY 1087 /*! cache: forced eviction - pages evicted that were dirty time (usecs) */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1087 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DIRTY_TIME 1088 /*! * cache: forced eviction - pages selected because of a large number of * updates to a single item */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_LONG_UPDATE_LIST 1088 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_LONG_UPDATE_LIST 1089 /*! * cache: forced eviction - pages selected because of too many deleted * items count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1089 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1090 /*! cache: forced eviction - pages selected count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1090 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1091 /*! cache: forced eviction - pages selected unable to be evicted count */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1091 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1092 /*! cache: forced eviction - pages selected unable to be evicted time */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1092 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1093 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1093 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1094 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1094 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1095 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1095 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1096 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1096 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1097 /*! cache: history store score */ -#define WT_STAT_CONN_CACHE_HS_SCORE 1097 +#define WT_STAT_CONN_CACHE_HS_SCORE 1098 /*! cache: history store table insert calls */ -#define WT_STAT_CONN_CACHE_HS_INSERT 1098 +#define WT_STAT_CONN_CACHE_HS_INSERT 1099 /*! cache: history store table insert calls that returned restart */ -#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1099 +#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1100 /*! cache: history store table max on-disk size */ -#define WT_STAT_CONN_CACHE_HS_ONDISK_MAX 1100 +#define WT_STAT_CONN_CACHE_HS_ONDISK_MAX 1101 /*! cache: history store table on-disk size */ -#define WT_STAT_CONN_CACHE_HS_ONDISK 1101 +#define WT_STAT_CONN_CACHE_HS_ONDISK 1102 /*! * cache: history store table out-of-order resolved updates that lose * their durable timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1102 +#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1103 /*! * cache: history store table out-of-order updates that were fixed up by * reinserting with the fixed timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_REINSERT 1103 +#define WT_STAT_CONN_CACHE_HS_ORDER_REINSERT 1104 /*! cache: history store table reads */ -#define WT_STAT_CONN_CACHE_HS_READ 1104 +#define WT_STAT_CONN_CACHE_HS_READ 1105 /*! cache: history store table reads missed */ -#define WT_STAT_CONN_CACHE_HS_READ_MISS 1105 +#define WT_STAT_CONN_CACHE_HS_READ_MISS 1106 /*! cache: history store table reads requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1106 +#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1107 /*! * cache: history store table truncation by rollback to stable to remove * an unstable update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1107 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1108 /*! * cache: history store table truncation by rollback to stable to remove * an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1108 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1109 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1109 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1110 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1110 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1111 /*! * cache: history store table truncation to remove range of updates due * to out-of-order timestamp update on data page */ -#define WT_STAT_CONN_CACHE_HS_ORDER_REMOVE 1111 +#define WT_STAT_CONN_CACHE_HS_ORDER_REMOVE 1112 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1112 +#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1113 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1113 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1114 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1114 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1115 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1115 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1116 /*! cache: internal pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1116 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1117 /*! cache: internal pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1117 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1118 /*! cache: internal pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1118 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1119 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1119 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1120 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1120 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1121 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1121 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1122 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1122 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1123 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1123 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1124 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1124 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1125 /*! cache: operations timed out waiting for space in cache */ -#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1125 +#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1126 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1126 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1127 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1127 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1128 /*! cache: page written requiring history store records */ -#define WT_STAT_CONN_CACHE_WRITE_HS 1128 +#define WT_STAT_CONN_CACHE_WRITE_HS 1129 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1129 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1130 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1130 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1131 /*! cache: pages evicted in parallel with checkpoint */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_IN_PARALLEL_WITH_CHECKPOINT 1131 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_IN_PARALLEL_WITH_CHECKPOINT 1132 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1132 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1133 /*! cache: pages queued for eviction post lru sorting */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1133 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1134 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1134 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1135 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1135 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1136 /*! * cache: pages queued for urgent eviction from history store due to high * dirty content */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT_HS_DIRTY 1136 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT_HS_DIRTY 1137 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1137 +#define WT_STAT_CONN_CACHE_READ 1138 /*! cache: pages read into cache after truncate */ -#define WT_STAT_CONN_CACHE_READ_DELETED 1138 +#define WT_STAT_CONN_CACHE_READ_DELETED 1139 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1139 +#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1140 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1140 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1141 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1141 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1142 /*! cache: pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1142 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1143 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1143 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1144 /*! * cache: pages selected for eviction unable to be evicted as the parent * page has overflow items */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1144 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1145 /*! * cache: pages selected for eviction unable to be evicted because of * active children on an internal page */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1145 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1146 /*! * cache: pages selected for eviction unable to be evicted because of * failure in reconciliation */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1146 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1147 /*! * cache: pages selected for eviction unable to be evicted because of * race between checkpoint and out of order timestamps handling */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_CHECKPOINT_OUT_OF_ORDER_TS 1147 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_CHECKPOINT_OUT_OF_ORDER_TS 1148 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1148 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1149 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1149 +#define WT_STAT_CONN_CACHE_WRITE 1150 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1150 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1151 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1151 +#define WT_STAT_CONN_CACHE_OVERHEAD 1152 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1152 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1153 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1153 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1154 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1154 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1155 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1155 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1156 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1156 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1157 /*! capacity: background fsync file handles considered */ -#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1157 +#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1158 /*! capacity: background fsync file handles synced */ -#define WT_STAT_CONN_FSYNC_ALL_FH 1158 +#define WT_STAT_CONN_FSYNC_ALL_FH 1159 /*! capacity: background fsync time (msecs) */ -#define WT_STAT_CONN_FSYNC_ALL_TIME 1159 +#define WT_STAT_CONN_FSYNC_ALL_TIME 1160 /*! capacity: bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1160 +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1161 /*! capacity: bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1161 +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1162 /*! capacity: bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1162 +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1163 /*! capacity: bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1163 +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1164 /*! capacity: bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1164 +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1165 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1165 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1166 /*! capacity: time waiting due to total capacity (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1166 +#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1167 /*! capacity: time waiting during checkpoint (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1167 +#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1168 /*! capacity: time waiting during eviction (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1168 +#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1169 /*! capacity: time waiting during logging (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_LOG 1169 +#define WT_STAT_CONN_CAPACITY_TIME_LOG 1170 /*! capacity: time waiting during read (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_READ 1170 +#define WT_STAT_CONN_CAPACITY_TIME_READ 1171 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_CONN_CC_PAGES_EVICT 1171 +#define WT_STAT_CONN_CC_PAGES_EVICT 1172 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_CONN_CC_PAGES_REMOVED 1172 +#define WT_STAT_CONN_CC_PAGES_REMOVED 1173 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1173 +#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1174 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_CONN_CC_PAGES_VISITED 1174 +#define WT_STAT_CONN_CC_PAGES_VISITED 1175 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1175 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1176 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1176 +#define WT_STAT_CONN_COND_AUTO_WAIT 1177 /*! * connection: auto adjusting condition wait raced to update timeout and * skipped updating */ -#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1177 +#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1178 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1178 +#define WT_STAT_CONN_TIME_TRAVEL 1179 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1179 +#define WT_STAT_CONN_FILE_OPEN 1180 /*! connection: hash bucket array size for data handles */ -#define WT_STAT_CONN_BUCKETS_DH 1180 +#define WT_STAT_CONN_BUCKETS_DH 1181 /*! connection: hash bucket array size general */ -#define WT_STAT_CONN_BUCKETS 1181 +#define WT_STAT_CONN_BUCKETS 1182 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1182 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1183 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1183 +#define WT_STAT_CONN_MEMORY_FREE 1184 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1184 +#define WT_STAT_CONN_MEMORY_GROW 1185 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1185 +#define WT_STAT_CONN_COND_WAIT 1186 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1186 +#define WT_STAT_CONN_RWLOCK_READ 1187 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1187 +#define WT_STAT_CONN_RWLOCK_WRITE 1188 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1188 +#define WT_STAT_CONN_FSYNC_IO 1189 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1189 +#define WT_STAT_CONN_READ_IO 1190 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1190 +#define WT_STAT_CONN_WRITE_IO 1191 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1191 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1192 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1192 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1193 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1193 +#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1194 /*! * cursor: Total number of pages skipped without reading by cursor next * calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_PAGE_COUNT 1194 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_PAGE_COUNT 1195 /*! * cursor: Total number of pages skipped without reading by cursor prev * calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_PAGE_COUNT 1195 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_PAGE_COUNT 1196 /*! * cursor: Total number of times a search near has exited due to prefix * config */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 1196 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 1197 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1197 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1198 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1198 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1199 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1199 +#define WT_STAT_CONN_CURSOR_CACHE 1200 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1200 +#define WT_STAT_CONN_CURSOR_CREATE 1201 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1201 +#define WT_STAT_CONN_CURSOR_INSERT 1202 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1202 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1203 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1203 +#define WT_STAT_CONN_CURSOR_MODIFY 1204 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1204 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1205 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1205 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1206 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1206 +#define WT_STAT_CONN_CURSOR_NEXT 1207 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1207 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1208 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1208 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1209 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1209 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1210 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1210 +#define WT_STAT_CONN_CURSOR_RESTART 1211 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1211 +#define WT_STAT_CONN_CURSOR_PREV 1212 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1212 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1213 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1213 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1214 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1214 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1215 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1215 +#define WT_STAT_CONN_CURSOR_REMOVE 1216 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1216 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1217 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1217 +#define WT_STAT_CONN_CURSOR_RESERVE 1218 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1218 +#define WT_STAT_CONN_CURSOR_RESET 1219 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1219 +#define WT_STAT_CONN_CURSOR_SEARCH 1220 /*! cursor: cursor search history store calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_HS 1220 +#define WT_STAT_CONN_CURSOR_SEARCH_HS 1221 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1221 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1222 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1222 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1223 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1223 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1224 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1224 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1225 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1225 +#define WT_STAT_CONN_CURSOR_SWEEP 1226 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1226 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1227 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1227 +#define WT_STAT_CONN_CURSOR_UPDATE 1228 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1228 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1229 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1229 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1230 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1230 +#define WT_STAT_CONN_CURSOR_REOPEN 1231 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1231 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1232 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1232 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1233 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1233 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1234 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1234 +#define WT_STAT_CONN_DH_SWEEP_REF 1235 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1235 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1236 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1236 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1237 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1237 +#define WT_STAT_CONN_DH_SWEEP_TOD 1238 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1238 +#define WT_STAT_CONN_DH_SWEEPS 1239 /*! * data-handle: connection sweeps skipped due to checkpoint gathering * handles */ -#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1239 +#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1240 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1240 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1241 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1241 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1242 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1242 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1243 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1243 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1244 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1244 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1245 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1245 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1246 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1246 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1247 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1247 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1248 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1248 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1249 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1249 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1250 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1250 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1251 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1251 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1252 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1252 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1253 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1253 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1254 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1254 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1255 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1255 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1256 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1256 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1257 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1257 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1258 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1258 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1259 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1259 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1260 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1260 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1261 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1261 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1262 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1262 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1263 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1263 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1264 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1264 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1265 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1265 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1266 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1266 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1267 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1267 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1268 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1268 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1269 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1269 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1270 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1270 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1271 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1271 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1272 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1272 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1273 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1273 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1274 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1274 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1275 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1275 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1276 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1276 +#define WT_STAT_CONN_LOG_FLUSH 1277 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1277 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1278 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1278 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1279 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1279 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1280 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1280 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1281 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1281 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1282 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1282 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1283 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1283 +#define WT_STAT_CONN_LOG_SCANS 1284 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1284 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1285 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1285 +#define WT_STAT_CONN_LOG_WRITE_LSN 1286 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1286 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1287 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1287 +#define WT_STAT_CONN_LOG_SYNC 1288 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1288 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1289 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1289 +#define WT_STAT_CONN_LOG_SYNC_DIR 1290 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1290 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1291 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1291 +#define WT_STAT_CONN_LOG_WRITES 1292 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1292 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1293 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1293 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1294 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1294 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1295 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1295 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1296 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1296 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1297 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1297 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1298 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1298 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1299 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1299 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1300 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1300 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1301 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1301 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1302 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1302 +#define WT_STAT_CONN_LOG_SLOT_RACES 1303 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1303 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1304 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1304 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1305 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1305 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1306 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1306 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1307 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1307 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1308 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1308 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1309 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1309 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1310 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1310 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1311 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1311 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1312 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1312 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1313 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1313 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1314 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1314 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1315 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1315 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1316 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1316 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1317 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1317 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1318 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1318 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1319 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1319 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1320 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1320 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1321 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1321 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1322 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1322 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1323 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1323 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1324 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1324 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1325 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1325 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1326 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1326 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1327 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1327 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1328 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1328 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1329 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1329 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1330 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1330 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1331 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1331 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1332 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1332 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1333 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1333 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1334 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1334 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1335 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1335 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1336 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1336 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1337 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1337 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1338 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1338 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1339 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1339 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1340 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1340 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1341 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1341 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1342 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_CONN_REC_OVERFLOW_KEY_INTERNAL 1342 +#define WT_STAT_CONN_REC_OVERFLOW_KEY_INTERNAL 1343 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1343 +#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1344 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1344 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1345 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1345 +#define WT_STAT_CONN_REC_PAGES 1346 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1346 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1347 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1347 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1348 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1348 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1349 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1349 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1350 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1350 +#define WT_STAT_CONN_REC_PAGE_DELETE 1351 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1351 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1352 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1352 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1353 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1353 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1354 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1354 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1355 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1355 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1356 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1356 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1357 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1357 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1358 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1358 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1359 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1359 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1360 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1360 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1361 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1361 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1362 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1362 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1363 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1363 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1364 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1364 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1365 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1365 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1366 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1366 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1367 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1367 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1368 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1368 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1369 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1369 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1370 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1370 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1371 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1371 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1372 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1372 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1373 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1373 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1374 /*! session: flush state races */ -#define WT_STAT_CONN_FLUSH_STATE_RACES 1374 +#define WT_STAT_CONN_FLUSH_STATE_RACES 1375 /*! session: flush_tier operation calls */ -#define WT_STAT_CONN_FLUSH_TIER 1375 +#define WT_STAT_CONN_FLUSH_TIER 1376 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1376 +#define WT_STAT_CONN_SESSION_OPEN 1377 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1377 +#define WT_STAT_CONN_SESSION_QUERY_TS 1378 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1378 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1379 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1379 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1380 +/*! session: table alter triggering checkpoint calls */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_TRIGGER_CHECKPOINT 1381 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1380 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1382 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1381 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1383 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1382 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1384 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1383 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1385 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1384 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1386 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1385 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1387 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1386 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1388 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1387 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1389 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1388 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1390 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1389 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1391 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1390 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1392 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1391 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1393 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1392 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1394 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1393 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1395 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1394 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1396 /*! session: tiered operations dequeued and processed */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1395 +#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1397 /*! session: tiered operations scheduled */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1396 +#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1398 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_CONN_TIERED_RETENTION 1397 +#define WT_STAT_CONN_TIERED_RETENTION 1399 /*! session: tiered storage object size */ -#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1398 +#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1400 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1399 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1401 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1400 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1402 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1401 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1403 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1402 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1404 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1403 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1405 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1404 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1406 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1405 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1407 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1406 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1408 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1407 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1409 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1408 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1410 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1409 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1411 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1410 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1412 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1411 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1413 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1412 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1414 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1413 +#define WT_STAT_CONN_PAGE_SLEEP 1415 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1414 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1416 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1415 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1417 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1416 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1418 /*! transaction: Number of prepared updates committed */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1417 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1419 /*! transaction: Number of prepared updates repeated on the same key */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1418 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1420 /*! transaction: Number of prepared updates rolled back */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1419 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1421 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1420 +#define WT_STAT_CONN_TXN_PREPARE 1422 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1421 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1423 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1422 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1424 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1423 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1425 /*! * transaction: prepared transactions rolled back and do not remove the * history store entry */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_DO_NOT_REMOVE_HS_UPDATE 1424 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_DO_NOT_REMOVE_HS_UPDATE 1426 /*! * transaction: prepared transactions rolled back and fix the history * store entry with checkpoint reserved transaction id */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_FIX_HS_UPDATE_WITH_CKPT_RESERVED_TXNID 1425 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_FIX_HS_UPDATE_WITH_CKPT_RESERVED_TXNID 1427 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1426 +#define WT_STAT_CONN_TXN_QUERY_TS 1428 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1427 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1429 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1428 +#define WT_STAT_CONN_TXN_RTS 1430 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1429 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1431 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1430 +#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1432 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1431 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1433 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1432 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1434 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1433 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1435 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1434 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1436 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1435 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1437 /*! transaction: rollback to stable skipping delete rle */ -#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1436 +#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1438 /*! transaction: rollback to stable skipping stable rle */ -#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1437 +#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1439 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1438 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1440 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1439 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1441 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1440 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1442 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1441 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1443 /*! transaction: sessions scanned in each walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1442 +#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1444 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1443 +#define WT_STAT_CONN_TXN_SET_TS 1445 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1444 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1446 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1445 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1447 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1446 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1448 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1447 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1449 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1448 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1450 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1449 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1451 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1450 +#define WT_STAT_CONN_TXN_BEGIN 1452 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1451 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1453 /*! * transaction: transaction checkpoint currently running for history * store file */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1452 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1454 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1453 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1455 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1454 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1456 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1455 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1457 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1456 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1458 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1457 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1459 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1458 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1460 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1461 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1462 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1461 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1463 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1462 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1464 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1463 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1465 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1464 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1466 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1465 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1467 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1466 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1468 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1467 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1469 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1468 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1470 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1469 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1471 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1470 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1472 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1471 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1473 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1472 +#define WT_STAT_CONN_TXN_CHECKPOINT 1474 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1473 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1475 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1474 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1476 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1475 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1477 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1476 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1478 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1477 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1479 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1478 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1480 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1479 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1481 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1480 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1482 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1481 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1483 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1482 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1484 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1483 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1485 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1484 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1486 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1485 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1487 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1486 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1488 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1487 +#define WT_STAT_CONN_TXN_COMMIT 1489 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1488 +#define WT_STAT_CONN_TXN_ROLLBACK 1490 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1489 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1491 /*! * @} diff --git a/src/third_party/wiredtiger/src/meta/meta_apply.c b/src/third_party/wiredtiger/src/meta/meta_apply.c index 668b24cff07..1d520a02a50 100644 --- a/src/third_party/wiredtiger/src/meta/meta_apply.c +++ b/src/third_party/wiredtiger/src/meta/meta_apply.c @@ -37,8 +37,7 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, continue; } - if (file_func == NULL || skip || - (!WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "tiered:"))) + if (file_func == NULL || skip || !WT_BTREE_PREFIX(uri)) continue; /* diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index d58d0149658..7bfc780a7cc 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -990,7 +990,7 @@ __wt_metadata_correct_base_write_gen(WT_SESSION_IMPL *session) while ((ret = cursor->next(cursor)) == 0) { WT_ERR(cursor->get_key(cursor, &uri)); - if (!WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "tiered:")) + if (!WT_BTREE_PREFIX(uri)) continue; WT_ERR(cursor->get_value(cursor, &config)); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 03f075e7cb0..10dd1d9daea 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -714,7 +714,7 @@ record_loop: for (n = 0; n < nrepeat; n += repeat_count, src_recno += repeat_count) { upd = NULL; if (ins != NULL && WT_INSERT_RECNO(ins) == src_recno) { - WT_ERR(__wt_rec_upd_select(session, r, ins, cip, vpack, &upd_select)); + WT_ERR(__wt_rec_upd_select(session, r, ins, NULL, vpack, &upd_select)); upd = upd_select.upd; ins = WT_SKIP_NEXT(ins); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 63b2c73c0d5..24b8e862dd2 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -25,7 +25,7 @@ __rec_update_stable(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_UPDATE *upd) * Save a WT_UPDATE list for later restoration. */ static inline int -__rec_update_save(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, void *ripcip, +__rec_update_save(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, WT_ROW *rip, WT_UPDATE *onpage_upd, bool supd_restore, size_t upd_memsize) { WT_SAVE_UPD *supd; @@ -36,11 +36,13 @@ __rec_update_save(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, voi WT_ASSERT(session, onpage_upd == NULL || onpage_upd->type == WT_UPDATE_STANDARD || onpage_upd->type == WT_UPDATE_MODIFY); + /* For columns, ins is never null, so rip == NULL implies ins != NULL. */ + WT_ASSERT(session, rip != NULL || ins != NULL); WT_RET(__wt_realloc_def(session, &r->supd_allocated, r->supd_next + 1, &r->supd)); supd = &r->supd[r->supd_next]; supd->ins = ins; - supd->ripcip = ripcip; + supd->rip = rip; supd->onpage_upd = onpage_upd; supd->restore = supd_restore; ++r->supd_next; @@ -381,7 +383,7 @@ __rec_validate_upd_chain(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_UPDATE *s * Return the update in a list that should be written (or NULL if none can be written). */ int -__wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, void *ripcip, +__wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, WT_ROW *rip, WT_CELL_UNPACK_KV *vpack, WT_UPDATE_SELECT *upd_select) { WT_DECL_ITEM(tmp); @@ -419,8 +421,12 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v */ if (ins != NULL) first_upd = ins->upd; - else if ((first_upd = WT_ROW_UPDATE(page, ripcip)) == NULL) - return (0); + else { + /* Note: ins is never null for columns. */ + WT_ASSERT(session, rip != NULL && page->type == WT_PAGE_ROW_LEAF); + if ((first_upd = WT_ROW_UPDATE(page, rip)) == NULL) + return (0); + } for (upd = first_upd; upd != NULL; upd = upd->next) { if ((txnid = upd->txnid) == WT_TXN_ABORTED) @@ -722,7 +728,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v (has_newer_updates || F_ISSET(S2C(session), WT_CONN_IN_MEMORY) || page->type == WT_PAGE_COL_FIX); - WT_ERR(__rec_update_save(session, r, ins, ripcip, onpage_upd, supd_restore, upd_memsize)); + WT_ERR(__rec_update_save(session, r, ins, rip, onpage_upd, supd_restore, upd_memsize)); /* * Mark the selected update (and potentially the tombstone preceding it) as being destined diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 03b5860c22e..a4f006b82c3 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -1069,7 +1069,7 @@ __rec_split_row_promote(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_ITEM *key, for (i = r->supd_next; i > 0; --i) { supd = &r->supd[i - 1]; if (supd->ins == NULL) - WT_ERR(__wt_row_leaf_key(session, r->page, supd->ripcip, update, false)); + WT_ERR(__wt_row_leaf_key(session, r->page, supd->rip, update, false)); else { update->data = WT_INSERT_KEY(supd->ins); update->size = WT_INSERT_KEY_SIZE(supd->ins); @@ -1478,7 +1478,7 @@ __rec_split_write_supd( for (i = 0, supd = r->supd; i < r->supd_next; ++i, ++supd) { if (supd->ins == NULL) - WT_ERR(__wt_row_leaf_key(session, page, supd->ripcip, key, false)); + WT_ERR(__wt_row_leaf_key(session, page, supd->rip, key, false)); else { key->data = WT_INSERT_KEY(supd->ins); key->size = WT_INSERT_KEY_SIZE(supd->ins); @@ -1504,9 +1504,8 @@ __rec_split_write_supd( for (j = 0; i < r->supd_next; ++j, ++i) { /* Account for the remaining update memory. */ if (r->supd[i].ins == NULL) - upd = page->modify->mod_row_update[page->type == WT_PAGE_ROW_LEAF ? - WT_ROW_SLOT(page, r->supd[i].ripcip) : - WT_COL_SLOT(page, r->supd[i].ripcip)]; + /* Note: ins is never NULL for column-store */ + upd = page->modify->mod_row_update[WT_ROW_SLOT(page, r->supd[i].rip)]; else upd = r->supd[i].ins->upd; r->supd_memsize += __wt_update_list_memsize(upd); diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 62464af3c6a..5a4320b493e 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -142,7 +142,7 @@ __wt_session_copy_values(WT_SESSION_IMPL *session) WT_TXN_SHARED *txn_shared = WT_SESSION_TXN_SHARED(session); WT_ASSERT(session, txn_shared->pinned_id != WT_TXN_NONE || - (WT_PREFIX_MATCH(cursor->uri, "file:") && + (WT_BTREE_PREFIX(cursor->uri) && F_ISSET((WT_CURSOR_BTREE *)cursor, WT_CBT_NO_TXN))); #endif WT_RET(__cursor_localvalue(cursor)); @@ -637,16 +637,13 @@ err: } /* - * __session_alter -- - * Alter a table setting. + * __session_alter_internal -- + * Internal implementation of the WT_SESSION.alter method. */ static int -__session_alter(WT_SESSION *wt_session, const char *uri, const char *config) +__session_alter_internal(WT_SESSION_IMPL *session, const char *uri, const char *config) { WT_DECL_RET; - WT_SESSION_IMPL *session; - - session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, alter, config, cfg); @@ -675,6 +672,78 @@ err: } /* + * __wt_session_blocking_checkpoint -- + * Perform a checkpoint or wait if it is already running to resolve an EBUSY error. + */ +int +__wt_session_blocking_checkpoint(WT_SESSION_IMPL *session, bool force, uint64_t seconds) +{ + WT_DECL_RET; + WT_TXN_GLOBAL *txn_global; + uint64_t txn_gen; + const char *cfg[3] = {NULL, NULL, NULL}; + + cfg[0] = WT_CONFIG_BASE(session, WT_SESSION_checkpoint); + if (force) + cfg[1] = "force=1"; + + if ((ret = __wt_txn_checkpoint(session, cfg, false)) == 0) + return (0); + WT_RET_BUSY_OK(ret); + + /* + * If there's a checkpoint running, wait for it to complete. If there's no checkpoint running or + * the checkpoint generation number changes, the checkpoint blocking us has completed. + */ +#define WT_CKPT_WAIT 2 + txn_global = &S2C(session)->txn_global; + for (txn_gen = __wt_gen(session, WT_GEN_CHECKPOINT);; __wt_sleep(WT_CKPT_WAIT, 0)) { + /* + * This loop only checks objects that are declared volatile, therefore no barriers are + * needed. + */ + if (!txn_global->checkpoint_running || txn_gen != __wt_gen(session, WT_GEN_CHECKPOINT)) + break; + + /* If there's a timeout, give up. */ + if (seconds == 0) + continue; + if (seconds <= WT_CKPT_WAIT) + return (EBUSY); + seconds -= WT_CKPT_WAIT; + } + + return (0); +} + +/* + * __session_alter -- + * Alter a table setting. + */ +static int +__session_alter(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + + /* + * Alter table can return EBUSY error when the table is modified in parallel by eviction. Retry + * the command after performing a system wide checkpoint. Only retry once to avoid potentially + * waiting forever. + */ + ret = __session_alter_internal(session, uri, config); + if (ret == EBUSY) { + WT_RET(__wt_session_blocking_checkpoint(session, false, 0)); + WT_STAT_CONN_INCR(session, session_table_alter_trigger_checkpoint); + ret = __session_alter_internal(session, uri, config); + } + + return (ret); +} + +/* * __session_alter_readonly -- * WT_SESSION->alter method; readonly version. */ @@ -1254,7 +1323,7 @@ __wt_session_range_truncate( local_start = false; if (uri != NULL) { - WT_ASSERT(session, WT_PREFIX_MATCH(uri, "file:")); + WT_ASSERT(session, WT_BTREE_PREFIX(uri)); /* * A URI file truncate becomes a range truncate where we set a start cursor at the * beginning. We already know the NULL stop goes to the end of the range. @@ -1402,7 +1471,7 @@ __session_truncate( WT_ERR_MSG(session, EINVAL, "the truncate method should not specify any target after the log: URI prefix"); WT_ERR(__wt_log_truncate_files(session, start, false)); - } else if (WT_PREFIX_MATCH(uri, "file:")) + } else if (WT_BTREE_PREFIX(uri)) WT_ERR(__wt_session_range_truncate(session, uri, start, stop)); else /* Wait for checkpoints to avoid EBUSY errors. */ diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index b4064eb1f69..1f83d6e4c83 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -193,53 +193,6 @@ __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) } /* - * __compact_checkpoint -- - * Perform a checkpoint for compaction. - */ -static int -__compact_checkpoint(WT_SESSION_IMPL *session) -{ - WT_DECL_RET; - WT_TXN_GLOBAL *txn_global; - uint64_t txn_gen; - - /* - * Force compaction checkpoints: we don't want to skip it because the work we need to have done - * is done in the underlying block manager. - */ - const char *checkpoint_cfg[] = { - WT_CONFIG_BASE(session, WT_SESSION_checkpoint), "force=1", NULL}; - - /* Checkpoints take a lot of time, check if we've run out. */ - WT_RET(__wt_session_compact_check_timeout(session)); - - ret = __wt_txn_checkpoint(session, checkpoint_cfg, false); - if (ret == 0) - return (0); - WT_RET_BUSY_OK(ret); - - /* - * If there's a checkpoint running, wait for it to complete, checking if we're out of time. If - * there's no checkpoint running or the checkpoint generation number changes, the checkpoint - * blocking us has completed. - */ - txn_global = &S2C(session)->txn_global; - for (txn_gen = __wt_gen(session, WT_GEN_CHECKPOINT);;) { - /* - * This loop only checks objects that are declared volatile, therefore no barriers are - * needed. - */ - if (!txn_global->checkpoint_running || txn_gen != __wt_gen(session, WT_GEN_CHECKPOINT)) - break; - - WT_RET(__wt_session_compact_check_timeout(session)); - __wt_sleep(2, 0); - } - - return (0); -} - -/* * __compact_worker -- * Function to alternate between checkpoints and compaction calls. */ @@ -260,7 +213,7 @@ __compact_worker(WT_SESSION_IMPL *session) /* * Perform an initial checkpoint (see this file's leading comment for details). */ - WT_ERR(__compact_checkpoint(session)); + WT_ERR(__wt_session_blocking_checkpoint(session, true, session->compact->max_time)); /* * We compact 10% of a file on each pass (but the overall size of the file is decreasing each @@ -310,8 +263,8 @@ __compact_worker(WT_SESSION_IMPL *session) /* * Perform two checkpoints (see this file's leading comment for details). */ - WT_ERR(__compact_checkpoint(session)); - WT_ERR(__compact_checkpoint(session)); + WT_ERR(__wt_session_blocking_checkpoint(session, true, session->compact->max_time)); + WT_ERR(__wt_session_blocking_checkpoint(session, true, session->compact->max_time)); } err: diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index d224baab653..0376a34f998 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1075,6 +1075,7 @@ static const char *const __stats_connection_desc[] = { "cache: eviction server unable to reach eviction goal", "cache: eviction server waiting for a leaf page", "cache: eviction state", + "cache: eviction walk most recent sleeps for checkpoint handle gathering", "cache: eviction walk target pages histogram - 0-9", "cache: eviction walk target pages histogram - 10-31", "cache: eviction walk target pages histogram - 128 and higher", @@ -1408,6 +1409,7 @@ static const char *const __stats_connection_desc[] = { "session: session query timestamp calls", "session: table alter failed calls", "session: table alter successful calls", + "session: table alter triggering checkpoint calls", "session: table alter unchanged and skipped", "session: table compact failed calls", "session: table compact successful calls", @@ -1617,6 +1619,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_slow = 0; stats->cache_eviction_walk_leaf_notfound = 0; /* not clearing cache_eviction_state */ + stats->cache_eviction_walk_sleeps = 0; stats->cache_eviction_target_page_lt10 = 0; stats->cache_eviction_target_page_lt32 = 0; stats->cache_eviction_target_page_ge128 = 0; @@ -1941,6 +1944,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->session_query_ts = 0; /* not clearing session_table_alter_fail */ /* not clearing session_table_alter_success */ + /* not clearing session_table_alter_trigger_checkpoint */ /* not clearing session_table_alter_skip */ /* not clearing session_table_compact_fail */ /* not clearing session_table_compact_success */ @@ -2128,6 +2132,7 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cache_eviction_slow += WT_STAT_READ(from, cache_eviction_slow); to->cache_eviction_walk_leaf_notfound += WT_STAT_READ(from, cache_eviction_walk_leaf_notfound); to->cache_eviction_state += WT_STAT_READ(from, cache_eviction_state); + to->cache_eviction_walk_sleeps += WT_STAT_READ(from, cache_eviction_walk_sleeps); to->cache_eviction_target_page_lt10 += WT_STAT_READ(from, cache_eviction_target_page_lt10); to->cache_eviction_target_page_lt32 += WT_STAT_READ(from, cache_eviction_target_page_lt32); to->cache_eviction_target_page_ge128 += WT_STAT_READ(from, cache_eviction_target_page_ge128); @@ -2486,6 +2491,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->session_query_ts += WT_STAT_READ(from, session_query_ts); to->session_table_alter_fail += WT_STAT_READ(from, session_table_alter_fail); to->session_table_alter_success += WT_STAT_READ(from, session_table_alter_success); + to->session_table_alter_trigger_checkpoint += + WT_STAT_READ(from, session_table_alter_trigger_checkpoint); to->session_table_alter_skip += WT_STAT_READ(from, session_table_alter_skip); to->session_table_compact_fail += WT_STAT_READ(from, session_table_compact_fail); to->session_table_compact_success += WT_STAT_READ(from, session_table_compact_success); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_config.c b/src/third_party/wiredtiger/src/tiered/tiered_config.c index 5c88ea1adaa..611ea8323c1 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_config.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_config.c @@ -62,8 +62,9 @@ __wt_tiered_bucket_config( WT_SESSION_IMPL *session, const char *cfg[], WT_BUCKET_STORAGE **bstoragep) { WT_BUCKET_STORAGE *bstorage, *new; - WT_CONFIG_ITEM auth, bucket, name, prefix; + WT_CONFIG_ITEM auth, bucket, cachedir, name, prefix; WT_CONNECTION_IMPL *conn; + WT_DECL_ITEM(buf); WT_DECL_RET; WT_NAMED_STORAGE_SOURCE *nstorage; WT_STORAGE_SOURCE *storage; @@ -72,6 +73,7 @@ __wt_tiered_bucket_config( *bstoragep = NULL; WT_RET(__wt_config_gets(session, cfg, "tiered_storage.name", &name)); + WT_RET(__wt_scr_alloc(session, 0, &buf)); bstorage = new = NULL; conn = S2C(session); @@ -92,14 +94,15 @@ __wt_tiered_bucket_config( if (conn->bstorage == NULL && bstoragep != &conn->bstorage) WT_ERR_MSG( session, EINVAL, "table tiered storage requires connection tiered storage to be set"); - /* A bucket and bucket_prefix are required, auth_token is not. */ + /* A bucket and bucket_prefix are required, cache_directory and auth_token are not. */ + WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.auth_token", &auth)); WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.bucket", &bucket)); if (bucket.len == 0) WT_ERR_MSG(session, EINVAL, "table tiered storage requires bucket to be set"); WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.bucket_prefix", &prefix)); if (prefix.len == 0) WT_ERR_MSG(session, EINVAL, "table tiered storage requires bucket_prefix to be set"); - WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.auth_token", &auth)); + WT_ERR(__wt_config_gets(session, cfg, "tiered_storage.cache_directory", &cachedir)); hash = __wt_hash_city64(bucket.str, bucket.len); hash_bucket = hash & (conn->hash_size - 1); @@ -115,10 +118,13 @@ __wt_tiered_bucket_config( WT_ERR(__wt_strndup(session, auth.str, auth.len, &new->auth_token)); WT_ERR(__wt_strndup(session, bucket.str, bucket.len, &new->bucket)); WT_ERR(__wt_strndup(session, prefix.str, prefix.len, &new->bucket_prefix)); + WT_ERR(__wt_strndup(session, cachedir.str, auth.len, &new->cache_directory)); storage = nstorage->storage_source; + if (cachedir.len != 0) + WT_ERR(__wt_buf_fmt(session, buf, "cache_directory=%s", new->cache_directory)); WT_ERR(storage->ss_customize_file_system( - storage, &session->iface, new->bucket, new->auth_token, NULL, &new->file_system)); + storage, &session->iface, new->bucket, new->auth_token, buf->data, &new->file_system)); new->storage_source = storage; /* If we're creating a new bucket storage, parse the other settings into it. */ @@ -138,6 +144,7 @@ err: __wt_free(session, new); } __wt_spin_unlock(session, &conn->storage_lock); + __wt_scr_free(session, &buf); return (ret); } @@ -181,6 +188,7 @@ err: __wt_free(session, conn->bstorage->auth_token); __wt_free(session, conn->bstorage->bucket); __wt_free(session, conn->bstorage->bucket_prefix); + __wt_free(session, conn->bstorage->cache_directory); __wt_free(session, conn->bstorage); return (ret); } diff --git a/src/third_party/wiredtiger/src/tiered/tiered_opener.c b/src/third_party/wiredtiger/src/tiered/tiered_opener.c index e65d52e4694..5ca027510fc 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_opener.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_opener.c @@ -45,6 +45,19 @@ __tiered_opener_open(WT_BLOCK_FILE_OPENER *opener, WT_SESSION_IMPL *session, uin flags |= WT_FS_OPEN_READONLY; WT_WITH_BUCKET_STORAGE( bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); + if (ret == ENOENT) { + /* + * There is a window where the object may not be copied yet to the bucket. If it isn't + * found try the local system. If it isn't found there then try the bucket one more + * time. + */ + ret = __wt_open(session, object_name, type, flags, fhp); + __wt_errx(session, "OPENER: local %s ret %d", object_name, ret); + if (ret == ENOENT) + WT_WITH_BUCKET_STORAGE( + bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); + WT_ERR(ret); + } } err: __wt_free(session, object_uri); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 29c183789e4..754860845ad 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -140,42 +140,43 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) /* * __wt_txn_user_active -- - * Check whether there are any running user transactions. Note that a new transactions may start - * on a session we have already examined and the caller needs to be aware of this limitation. - * Exclude prepared user transactions from this check. + * Check whether there are any running user transactions. Note that a new transaction may start + * after we return from this call and therefore caller should be aware of this limitation. */ bool __wt_txn_user_active(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_SESSION_IMPL *session_in_list; + WT_TXN_GLOBAL *txn_global; + WT_TXN_SHARED *txn_shared; uint32_t i, session_cnt; bool txn_active; conn = S2C(session); txn_active = false; + txn_global = &conn->txn_global; + + /* We're going to scan the table: wait for the lock. */ + __wt_writelock(session, &txn_global->rwlock); - /* - * No lock is required because the session array is fixed size, but it may contain inactive - * entries. We must review any active session, so insert a read barrier after reading the active - * session count. That way, no matter what sessions come or go, we'll check the slots for all of - * the user sessions for active transactions when we started our check. - */ WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (i = 0, session_in_list = conn->sessions; i < session_cnt; i++, session_in_list++) { - /* Skip inactive sessions. */ - if (!session_in_list->active) + WT_STAT_CONN_INCR(session, txn_walk_sessions); + for (i = 0, session_in_list = conn->sessions, txn_shared = txn_global->txn_shared_list; + i < session_cnt; i++, session_in_list++, txn_shared++) { + + /* Skip inactive or internal sessions. */ + if (!session_in_list->active || F_ISSET(session_in_list, WT_SESSION_INTERNAL)) continue; - /* Check if a user session has a running transaction. Ignore prepared transactions. */ - if (F_ISSET(session_in_list->txn, WT_TXN_RUNNING) && - !F_ISSET(session_in_list, WT_SESSION_INTERNAL) && - !F_ISSET(session_in_list->txn, WT_TXN_PREPARE)) { + /* Check if a user session has a running transaction. */ + if (txn_shared->id != WT_TXN_NONE || txn_shared->pinned_id != WT_TXN_NONE) { txn_active = true; break; } } + __wt_writeunlock(session, &txn_global->rwlock); return (txn_active); } @@ -1008,10 +1009,19 @@ __txn_fixup_prepared_update( * Transaction error and prepare are cleared temporarily as cursor functions are not allowed * after an error or a prepared transaction. */ - txn_flags = FLD_MASK(txn->flags, WT_TXN_ERROR | WT_TXN_PREPARE); + txn_flags = FLD_MASK(txn->flags, WT_TXN_ERROR); F_CLR(txn, txn_flags); /* + * The API layer will immediately return an error if the WT_TXN_PREPARE flag is set before + * attempting cursor operations. However, we can't clear the WT_TXN_PREPARE flag because a + * function in the eviction flow may attempt to forcibly rollback the transaction if it is not + * marked as a prepared transaction. The flag WT_TXN_PREPARE_IGNORE_API_CHECK is set so that + * cursor operations can proceed without having to clear the WT_TXN_PREPARE flag. + */ + F_SET(txn, WT_TXN_PREPARE_IGNORE_API_CHECK); + + /* * If the history update already has a stop time point and we are committing the prepared update * there is no work to do. */ @@ -1083,6 +1093,7 @@ __txn_fixup_prepared_update( err: F_SET(txn, txn_flags); + F_CLR(txn, WT_TXN_PREPARE_IGNORE_API_CHECK); return (ret); } @@ -1115,10 +1126,18 @@ __txn_search_prepared_op( } /* - * Transaction error and prepare are cleared temporarily as cursor functions are not allowed - * after an error or a prepared transaction. + * Transaction error is cleared temporarily as cursor functions are not allowed after an error. */ - txn_flags = FLD_MASK(txn->flags, WT_TXN_ERROR | WT_TXN_PREPARE); + txn_flags = FLD_MASK(txn->flags, WT_TXN_ERROR); + + /* + * The API layer will immediately return an error if the WT_TXN_PREPARE flag is set before + * attempting cursor operations. However, we can't clear the WT_TXN_PREPARE flag because a + * function in the eviction flow may attempt to forcibly rollback the transaction if it is not + * marked as a prepared transaction. The flag WT_TXN_PREPARE_IGNORE_API_CHECK is set so that + * cursor operations can proceed without having to clear the WT_TXN_PREPARE flag. + */ + F_SET(txn, WT_TXN_PREPARE_IGNORE_API_CHECK); switch (op->type) { case WT_TXN_OP_BASIC_COL: @@ -1142,6 +1161,7 @@ __txn_search_prepared_op( F_CLR(txn, txn_flags); WT_WITH_BTREE(session, op->btree, ret = __wt_btcur_search_prepared(cursor, updp)); F_SET(txn, txn_flags); + F_CLR(txn, WT_TXN_PREPARE_IGNORE_API_CHECK); WT_RET(ret); WT_RET_ASSERT(session, *updp != NULL, WT_NOTFOUND, "unable to locate update associated with a prepared operation"); @@ -1347,6 +1367,10 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_STAT_CONN_INCR(session, txn_prepared_updates_committed); } + /* Mark the page dirty once the prepared updates are resolved. */ + cbt = (WT_CURSOR_BTREE *)(*cursorp); + __wt_page_modify_set(session, cbt->ref->page); + /* * Fix the history store contents if they exist, when there are no more updates in the update * list. Only in eviction, it is possible to write an unfinished history store update when the diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 1c5ce6c32ee..86ba9274086 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -287,6 +287,8 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) * rollback error. We will ignore this dhandle as part of this checkpoint by returning * from here. */ + WT_IGNORE_RET(__wt_msg( + session, "WT_ROLLBACK: checkpoint raced with transaction operating on dhandle")); WT_TRET(__wt_metadata_cursor_release(session, &meta_cursor)); return (0); } diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index c19f63d84bc..04df57cf66f 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -313,17 +313,16 @@ __rollback_txn_visible_id(WT_SESSION_IMPL *session, uint64_t id) * satisfies the given timestamp. */ static int -__rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page, WT_COL *cip, - WT_ROW *rip, wt_timestamp_t rollback_timestamp, uint64_t recno) +__rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, uint64_t recno, + WT_ITEM *row_key, WT_CELL_UNPACK_KV *unpack, wt_timestamp_t rollback_timestamp) { - WT_CELL *kcell; - WT_CELL_UNPACK_KV *unpack, _unpack; WT_CURSOR *hs_cursor; WT_DECL_ITEM(full_value); WT_DECL_ITEM(hs_key); WT_DECL_ITEM(hs_value); WT_DECL_ITEM(key); WT_DECL_RET; + WT_PAGE *page; WT_TIME_WINDOW *hs_tw; WT_UPDATE *tombstone, *upd; wt_timestamp_t hs_durable_ts, hs_start_ts, hs_stop_durable_ts, newer_hs_durable_ts, pinned_ts; @@ -338,16 +337,8 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page bool first_record; #endif - /* - * Assert an exclusive or for rip and cip such that either only a cip for a column store or a - * rip for a row store are passed into the function. - */ - WT_ASSERT(session, (rip != NULL && cip == NULL) || (rip == NULL && cip != NULL)); + page = ref->page; - if (page == NULL) { - WT_ASSERT(session, ref != NULL); - page = ref->page; - } hs_cursor = NULL; tombstone = upd = NULL; hs_durable_ts = hs_start_ts = hs_stop_durable_ts = WT_TS_NONE; @@ -362,24 +353,19 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page WT_ERR(__wt_scr_alloc(session, 0, &hs_value)); if (rip != NULL) { - /* Unpack a row cell. */ - WT_ERR(__wt_scr_alloc(session, 0, &key)); - WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); - - /* Get the full update value from the data store. */ - unpack = &_unpack; - __wt_row_leaf_value_cell(session, page, rip, unpack); + if (row_key != NULL) + key = row_key; + else { + /* Unpack a row key. */ + WT_ERR(__wt_scr_alloc(session, 0, &key)); + WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); + } } else { - /* Unpack a column cell. */ + /* Manufacture a column key. */ WT_ERR(__wt_scr_alloc(session, WT_INTPACK64_MAXSIZE, &key)); memp = key->mem; WT_ERR(__wt_vpack_uint(&memp, 0, recno)); key->size = WT_PTRDIFF(memp, key->data); - - /* Get the full update value from the data store. */ - unpack = &_unpack; - kcell = WT_COL_PTR(page, cip); - __wt_cell_unpack_kv(session, page->dsk, kcell, unpack); } WT_ERR(__wt_scr_alloc(session, 0, &full_value)); @@ -642,7 +628,8 @@ err: __wt_scr_free(session, &full_value); __wt_scr_free(session, &hs_key); __wt_scr_free(session, &hs_value); - __wt_scr_free(session, &key); + if (rip == NULL || row_key == NULL) + __wt_scr_free(session, &key); if (hs_cursor != NULL) WT_TRET(hs_cursor->close(hs_cursor)); return (ret); @@ -653,11 +640,10 @@ err: * Fix the on-disk K/V version according to the given timestamp. */ static int -__rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, WT_ROW *rip, - wt_timestamp_t rollback_timestamp, uint64_t recno, bool *is_ondisk_stable) +__rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, uint64_t recno, + WT_ITEM *row_key, WT_CELL_UNPACK_KV *vpack, wt_timestamp_t rollback_timestamp, + bool *is_ondisk_stable) { - WT_CELL *kcell; - WT_CELL_UNPACK_KV *vpack, _vpack; WT_DECL_ITEM(tmp); WT_DECL_RET; WT_PAGE *page; @@ -666,26 +652,12 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W bool prepared; page = ref->page; - vpack = &_vpack; upd = NULL; /* Initialize the on-disk stable version flag. */ if (is_ondisk_stable != NULL) *is_ondisk_stable = false; - /* - * Assert an exclusive or for rip and cip such that either only a cip for a column store or a - * rip for a row store are passed into the function. - */ - WT_ASSERT(session, (rip != NULL && cip == NULL) || (rip == NULL && cip != NULL)); - - if (rip != NULL) - __wt_row_leaf_value_cell(session, page, rip, vpack); - else { - kcell = WT_COL_PTR(page, cip); - __wt_cell_unpack_kv(session, page->dsk, kcell, vpack); - } - prepared = vpack->tw.prepare; if (WT_IS_HS(session->dhandle)) { /* @@ -716,8 +688,8 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W __wt_timestamp_to_string(vpack->tw.start_ts, ts_string[1]), prepared ? "true" : "false", __wt_timestamp_to_string(rollback_timestamp, ts_string[2]), vpack->tw.start_txn); if (!F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - return ( - __rollback_ondisk_fixup_key(session, ref, NULL, cip, rip, rollback_timestamp, recno)); + return (__rollback_ondisk_fixup_key( + session, ref, rip, recno, row_key, vpack, rollback_timestamp)); else { /* * In-memory database don't have a history store to provide a stable update, so remove @@ -740,7 +712,7 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W WT_ASSERT(session, prepared == true); if (!F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) return (__rollback_ondisk_fixup_key( - session, ref, NULL, cip, rip, rollback_timestamp, recno)); + session, ref, rip, recno, row_key, vpack, rollback_timestamp)); else { /* * In-memory database don't have a history store to provide a stable update, so @@ -867,8 +839,8 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r WT_STAT_CONN_DATA_INCR(session, txn_rts_delete_rle_skipped); else { for (j = 0; j < rle; j++) { - WT_RET(__rollback_abort_ondisk_kv( - session, ref, cip, NULL, rollback_timestamp, recno + j, &is_ondisk_stable)); + WT_RET(__rollback_abort_ondisk_kv(session, ref, NULL, recno + j, NULL, &unpack, + rollback_timestamp, &is_ondisk_stable)); /* We can stop right away if the on-disk version is stable. */ if (is_ondisk_stable) { if (rle > 1) @@ -923,6 +895,7 @@ __rollback_abort_col_fix(WT_SESSION_IMPL *session, WT_PAGE *page, wt_timestamp_t static int __rollback_abort_row_leaf(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t rollback_timestamp) { + WT_CELL_UNPACK_KV *vpack, _vpack; WT_DECL_ITEM(key); WT_DECL_RET; WT_INSERT_HEAD *insert; @@ -930,7 +903,7 @@ __rollback_abort_row_leaf(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t WT_ROW *rip; WT_UPDATE *upd; uint32_t i; - bool stable_update_found; + bool have_key, stable_update_found; page = ref->page; @@ -952,7 +925,9 @@ __rollback_abort_row_leaf(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); WT_ERR( __rollback_abort_update(session, key, upd, rollback_timestamp, &stable_update_found)); - } + have_key = true; + } else + have_key = false; if ((insert = WT_ROW_INSERT(page, rip)) != NULL) WT_ERR(__rollback_abort_insert_list(session, page, insert, rollback_timestamp, NULL)); @@ -960,9 +935,12 @@ __rollback_abort_row_leaf(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t /* * If there is no stable update found in the update list, abort any on-disk value. */ - if (!stable_update_found) - WT_ERR( - __rollback_abort_ondisk_kv(session, ref, NULL, rip, rollback_timestamp, 0, NULL)); + if (!stable_update_found) { + vpack = &_vpack; + __wt_row_leaf_value_cell(session, page, rip, vpack); + WT_ERR(__rollback_abort_ondisk_kv( + session, ref, rip, 0, have_key ? key : NULL, vpack, rollback_timestamp, NULL)); + } } /* Mark the page as dirty to reconcile the page. */ @@ -1422,9 +1400,8 @@ __rollback_to_stable_btree_apply( bool dhandle_allocated, durable_ts_found, has_txn_updates_gt_than_ckpt_snap, perform_rts; bool prepared_updates; - /* Ignore non-file objects as well as the metadata and history store files. */ - if (!WT_PREFIX_MATCH(uri, "file:") || strcmp(uri, WT_HS_URI) == 0 || - strcmp(uri, WT_METAFILE_URI) == 0) + /* Ignore non-btree objects as well as the metadata and history store files. */ + if (!WT_BTREE_PREFIX(uri) || strcmp(uri, WT_HS_URI) == 0 || strcmp(uri, WT_METAFILE_URI) == 0) return (0); txn_global = &S2C(session)->txn_global; @@ -1576,7 +1553,7 @@ __wt_rollback_to_stable_one(WT_SESSION_IMPL *session, const char *uri, bool *ski * may contain, so set the caller's skip argument to true on all file objects, else set the * caller's skip argument to false so our caller continues down the tree of objects. */ - *skipp = WT_PREFIX_MATCH(uri, "file:"); + *skipp = WT_BTREE_PREFIX(uri); if (!*skipp) return (0); diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 9faa37d0c57..c138f23fe57 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -387,6 +387,13 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) last_oldest_ts = txn_global->oldest_timestamp; last_stable_ts = txn_global->stable_timestamp; + /* It is a no-op to set the oldest or stable timestamps behind the global values. */ + if (has_oldest && txn_global->has_oldest_timestamp && oldest_ts <= last_oldest_ts) + has_oldest = false; + + if (has_stable && txn_global->has_stable_timestamp && stable_ts <= last_stable_ts) + has_stable = false; + /* * First do error checking on the timestamp values. The oldest timestamp must always be less * than or equal to the stable timestamp. If we're only setting one then compare against the @@ -434,12 +441,6 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) __wt_readunlock(session, &txn_global->rwlock); /* Check if we are actually updating anything. */ - if (has_oldest && txn_global->has_oldest_timestamp && oldest_ts <= last_oldest_ts) - has_oldest = false; - - if (has_stable && txn_global->has_stable_timestamp && stable_ts <= last_stable_ts) - has_stable = false; - if (!has_durable && !has_oldest && !has_stable) return (0); diff --git a/src/third_party/wiredtiger/src/utilities/util_verify.c b/src/third_party/wiredtiger/src/utilities/util_verify.c index 910c03aa629..1ee4c1bcc90 100644 --- a/src/third_party/wiredtiger/src/utilities/util_verify.c +++ b/src/third_party/wiredtiger/src/utilities/util_verify.c @@ -67,7 +67,7 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) /* * The remaining argument is the table name. If we are verifying the history store we do not - * accept a URI. Otherwise, we need a URI top operate on. + * accept a URI. Otherwise, we need a URI to operate on. */ if (argc != 1) return (usage()); diff --git a/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh b/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh index fc98ff2f463..1cc0653aef9 100755 --- a/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh +++ b/src/third_party/wiredtiger/test/checkpoint/recovery-test.sh @@ -2,12 +2,25 @@ set -x -home=${1:-WT_TEST} +usage () { + cat << EOF +Usage: recovery_test.sh {config} {home directory} +EOF +} + +if [ "$#" -ne 2 ]; then + echo "Illegal number of parameters." + usage + exit 1 +fi + +config=$1 +home=$2 backup=$home.backup recovery=$home.recovery #./t -t r -W 3 -D -X -n 100000 -k 100000 -C cache_size=100MB -h $home > $home.out 2>&1 & -./t -t r -s 2 -m -W 3 -D -p -x -n 100000 -k 100000 -C cache_size=100MB -h $home > $home.out 2>&1 & +./t ${config} -h ${home} > $home.out 2>&1 & pid=$! trap "kill -9 $pid" 0 1 2 3 13 15 diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/burst_inserts_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/burst_inserts_default.txt new file mode 100644 index 00000000000..1f8efba08a3 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/burst_inserts_default.txt @@ -0,0 +1,4 @@ +# Configuration for burst_inserts. +# need to be defined. +duration_seconds=5, +cache_size_mb=250 diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/burst_inserts_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/burst_inserts_stress.txt new file mode 100644 index 00000000000..7dabeb9af7b --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/burst_inserts_stress.txt @@ -0,0 +1,41 @@ +# Used as a stress test for the framework. +duration_seconds=14400, +cache_size_mb=2000, +checkpoint_manager= +( + enabled=true, + op_rate=60s +), +runtime_monitor= +( + enabled=false +), +timestamp_manager= +( + enabled=true, + oldest_lag=10, + stable_lag=10 +), +workload_generator= +( + populate_config= + ( + collection_count=160, + key_count_per_collection=500, + key_size=200, + thread_count=40, + value_size=20000 + ), + insert_config= + ( + key_size=200, + op_rate=10s, + ops_per_transaction=(max=2000,min=1000), + thread_count=40, + value_size=20000 + ) +), +workload_tracking= +( + enabled=false, +) diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_default.txt index 5dc77f97645..f842cab4631 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_default.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_default.txt @@ -25,7 +25,7 @@ runtime_monitor= #10GB limit=10000000000 ), - postrun_statistics=[cache_hs_insert:0:100000000, cc_pages_removed:0:10000000], + postrun_statistics=[cache_hs_insert:10:100000000, cc_pages_removed:0:10000000], ), timestamp_manager= ( diff --git a/src/third_party/wiredtiger/test/cppsuite/create_test.sh b/src/third_party/wiredtiger/test/cppsuite/create_test.sh index 5076b43c7c3..00941911629 100755 --- a/src/third_party/wiredtiger/test/cppsuite/create_test.sh +++ b/src/third_party/wiredtiger/test/cppsuite/create_test.sh @@ -30,19 +30,19 @@ if test -f "$CONFIG"; then fi # Copy the default template. -cp tests/example_test.cxx $FILE +cp tests/example_test.cxx "$FILE" echo "Created $FILE." -cp configs/example_test_default.txt $CONFIG +cp configs/example_test_default.txt "$CONFIG" echo "Created $CONFIG." # Replace example_test with the new test name. SEARCH="example_test" -sed -i "s/$SEARCH/$1/" $FILE +sed -i "s/$SEARCH/$1/" "$FILE" echo "Updated $FILE." # Replace the first line of the configuration file. REPLACE="# Configuration for $1." -sed -i "1s/.*/$REPLACE/" $CONFIG +sed -i "1s/.*/$REPLACE/" "$CONFIG" echo "Updated $CONFIG." # Include the new test in run.cxx @@ -52,14 +52,14 @@ VALUE="#include \"$1.cxx\"" sed -i "/$SEARCH/a $VALUE" $FILE # Add the new test to the run_test() method -SEARCH="example_test(config, test_name).run()" -LINE_1="\ else if (test_name == \"$1\")\n" -LINE_2="\ $1(config, test_name).run();" +SEARCH="example_test(" +LINE_1="\else if (test_name == \"$1\")\n" +LINE_2="\ $1(test_harness::test_args{config, test_name, wt_open_config}).run();" sed -i "/$SEARCH/a $LINE_1$LINE_2" $FILE # Add the new test to all existing tests. -SEARCH="all_tests = {\"example_test\"" -REPLACE="$SEARCH, \"$1\"" +SEARCH="all_tests = {" +REPLACE="$SEARCH\"$1\", " sed -i "s/$SEARCH/$REPLACE/" $FILE echo "Updated $FILE." @@ -72,10 +72,10 @@ echo "Updated $FILE." # Trigger s_all echo "Running s_all.." -cd ../../dist +cd ../../dist || exit 1 ./s_all # Last changes to be done manually echo "Follow the next steps to execute your new test:" echo "1. Start editing $1.cxx" -echo "2. Compile your changes, go to build_posix/test/cppsuite and run your test with ./run -t $1" +echo "2. Compile your changes, go to <build_dir>/test/cppsuite and run your test with ./run -t $1" diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h index 875255e5969..8c5e2d17434 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.h @@ -80,10 +80,12 @@ class test : public database_operation { timestamp_manager *get_timestamp_manager(); thread_manager *get_thread_manager(); + protected: + configuration *_config; + private: const test_args &_args; std::vector<component *> _components; - configuration *_config; checkpoint_manager *_checkpoint_manager = nullptr; runtime_monitor *_runtime_monitor = nullptr; thread_manager *_thread_manager = nullptr; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx index ebd73acf286..2f85cab681e 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx @@ -49,16 +49,16 @@ populate_worker(thread_context *tc) * is closed, WiredTiger APIs close the cursors too. */ scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); - for (uint64_t i = 0; i < tc->key_count; ++i) { - /* Start a txn. */ + uint64_t j = 0; + while (j < tc->key_count) { tc->transaction.begin(); - if (tc->insert(cursor, coll.id, i)) { - /* We failed to insert, rollback our transaction and retry. */ + if (tc->insert(cursor, coll.id, j)) { + if (tc->transaction.commit()) { + ++j; + } + } else { tc->transaction.rollback(); - --i; - continue; } - tc->transaction.commit(); } } logger::log_msg(LOG_TRACE, "Populate: thread {" + std::to_string(tc->id) + "} finished"); @@ -156,33 +156,31 @@ database_operation::insert_operation(thread_context *tc) while (tc->running()) { uint64_t start_key = ccv[counter].coll.get_key_count(); uint64_t added_count = 0; - bool committed = true; tc->transaction.begin(); /* Collection cursor. */ auto &cc = ccv[counter]; while (tc->transaction.active() && tc->running()) { - /* Insert a key value pair. */ - bool rollback_required = tc->insert(cc.cursor, cc.coll.id, start_key + added_count); - if (!rollback_required) { + /* Insert a key value pair, rolling back the transaction if required. */ + if (!tc->insert(cc.cursor, cc.coll.id, start_key + added_count)) { + added_count = 0; + tc->transaction.rollback(); + } else { added_count++; if (tc->transaction.can_commit()) { - rollback_required = tc->transaction.commit(); - if (!rollback_required) + if (tc->transaction.commit()) { /* * We need to inform the database model that we've added these keys as some * other thread may rely on the key_count data. Only do so if we * successfully committed. */ cc.coll.increase_key_count(added_count); + } else { + added_count = 0; + } } } - if (rollback_required) { - added_count = 0; - tc->transaction.rollback(); - } - /* Sleep the duration defined by the op_rate. */ tc->sleep(); } @@ -191,6 +189,7 @@ database_operation::insert_operation(thread_context *tc) counter++; if (counter == collections_per_thread) counter = 0; + testutil_assert(counter < collections_per_thread); } /* Make sure the last transaction is rolled back now the work is finished. */ if (tc->transaction.active()) @@ -279,17 +278,16 @@ database_operation::update_operation(thread_context *tc) /* Choose a random key to update. */ uint64_t key_id = random_generator::instance().generate_integer<uint64_t>(0, coll.get_key_count() - 1); - bool rollback_required = tc->update(cursor, coll.id, tc->key_to_string(key_id)); + if (!tc->update(cursor, coll.id, tc->key_to_string(key_id))) { + tc->transaction.rollback(); + } /* Reset our cursor to avoid pinning content. */ testutil_check(cursor->reset(cursor.get())); /* Commit the current transaction if we're able to. */ - if (!rollback_required && tc->transaction.can_commit()) - rollback_required = tc->transaction.commit(); - - if (rollback_required) - tc->transaction.rollback(); + if (tc->transaction.can_commit()) + tc->transaction.commit(); } /* Make sure the last operation is rolled back now the work is finished. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx index 8a551b80a75..c539fbc34fc 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx @@ -27,6 +27,11 @@ */ #include "random_generator.h" +#include <algorithm> + +extern "C" { +#include "test_util.h" +} namespace test_harness { random_generator & @@ -37,25 +42,28 @@ random_generator::instance() } std::string -random_generator::generate_string(std::size_t length) +random_generator::generate_random_string(std::size_t length, characters_type type) { - std::string random_string; + std::string str; - for (std::size_t i = 0; i < length; ++i) - random_string += _characters[_distribution(_generator)]; + while (str.size() < length) + str += get_characters(type); - return (random_string); + std::shuffle(str.begin(), str.end(), _generator); + return (str.substr(0, length)); } std::string -random_generator::generate_pseudo_random_string(std::size_t length) +random_generator::generate_pseudo_random_string(std::size_t length, characters_type type) { std::string random_string; - std::size_t start_location = _distribution(_generator); + std::uniform_int_distribution<> &distribution = get_distribution(type); + std::size_t start_location = distribution(_generator); + const std::string &characters = get_characters(type); for (std::size_t i = 0; i < length; ++i) { - random_string += _characters[start_location]; - if (start_location == _characters.size() - 1) + random_string += characters[start_location]; + if (start_location == characters.size() - 1) start_location = 0; else start_location++; @@ -66,6 +74,40 @@ random_generator::generate_pseudo_random_string(std::size_t length) random_generator::random_generator() { _generator = std::mt19937(std::random_device{}()); - _distribution = std::uniform_int_distribution<>(0, _characters.size() - 1); + _alphanum_distrib = std::uniform_int_distribution<>(0, _pseudo_alphanum.size() - 1); + _alpha_distrib = std::uniform_int_distribution<>(0, _alphabet.size() - 1); } + +std::uniform_int_distribution<> & +random_generator::get_distribution(characters_type type) +{ + switch (type) { + case characters_type::ALPHABET: + return (_alpha_distrib); + break; + case characters_type::PSEUDO_ALPHANUMERIC: + return (_alphanum_distrib); + break; + default: + testutil_die(type, "Unexpected characters_type"); + break; + } +} + +const std::string & +random_generator::get_characters(characters_type type) +{ + switch (type) { + case characters_type::ALPHABET: + return (_alphabet); + break; + case characters_type::PSEUDO_ALPHANUMERIC: + return (_pseudo_alphanum); + break; + default: + testutil_die(type, "Unexpected characters_type"); + break; + } +} + } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h index cd6d7d4d7ec..31f44bbe98e 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h @@ -29,25 +29,36 @@ #ifndef RANDOM_GENERATOR_H #define RANDOM_GENERATOR_H +/* Following definitions are required in order to use printing format specifiers in C++. */ +#define __STDC_LIMIT_MACROS +#define __STDC_FORMAT_MACROS + #include <random> #include <string> namespace test_harness { /* Helper class to generate random values using uniform distributions. */ + +enum characters_type { PSEUDO_ALPHANUMERIC, ALPHABET }; + class random_generator { public: static random_generator &instance(); - public: /* No copies of the singleton allowed. */ random_generator(random_generator const &) = delete; random_generator &operator=(random_generator const &) = delete; /* Generate a random string of a given length. */ - std::string generate_string(std::size_t length); + std::string generate_random_string( + std::size_t length, characters_type type = PSEUDO_ALPHANUMERIC); - /* Generate a pseudo random string which compresses better. */ - std::string generate_pseudo_random_string(std::size_t length); + /* + * Generate a pseudo random string which compresses better. It should not be used to generate + * keys due to the limited randomness. + */ + std::string generate_pseudo_random_string( + std::size_t length, characters_type type = PSEUDO_ALPHANUMERIC); /* Generate a random integer between min and max. */ template <typename T> @@ -60,10 +71,13 @@ class random_generator { private: random_generator(); + std::uniform_int_distribution<> &get_distribution(characters_type type); + const std::string &get_characters(characters_type type); std::mt19937 _generator; - std::uniform_int_distribution<> _distribution; - const std::string _characters = + std::uniform_int_distribution<> _alphanum_distrib, _alpha_distrib; + const std::string _alphabet = "abcdefghijklmnopqrstuvwxyz"; + const std::string _pseudo_alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx index aadf992133e..f087d45cd06 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx @@ -29,6 +29,7 @@ #include "../core/configuration.h" #include "../timestamp_manager.h" #include "../util/api_const.h" +#include "../util/logger.h" #include "workload_tracking.h" #include "random_generator.h" #include "thread_context.h" @@ -81,22 +82,23 @@ transaction_context::try_begin(const std::string &config) begin(config); } -/* It's possible to receive rollback in commit which is handled internally. */ +/* + * It's possible to receive rollback in commit, when this happens the API will rollback the + * transaction internally. + */ bool transaction_context::commit(const std::string &config) { WT_DECL_RET; - testutil_assert(_in_txn); + testutil_assert(_in_txn && !_needs_rollback); if ((ret = _session->commit_transaction(_session, config.empty() ? nullptr : config.c_str())) != 0) { logger::log_msg(LOG_WARN, "Failed to commit transaction in commit, received error code: " + std::to_string(ret)); - _needs_rollback = true; - } else { - _op_count = 0; - _in_txn = false; } - return (_needs_rollback); + _op_count = 0; + _in_txn = false; + return (ret == 0); } void @@ -200,7 +202,7 @@ thread_context::update(scoped_cursor &cursor, uint64_t collection_id, const std: if (ret != 0) { if (ret == WT_ROLLBACK) { transaction.set_needs_rollback(true); - return (true); + return (false); } else testutil_die(ret, "unhandled error while trying to update a key"); } @@ -209,31 +211,39 @@ thread_context::update(scoped_cursor &cursor, uint64_t collection_id, const std: if (ret != 0) { if (ret == WT_ROLLBACK) { transaction.set_needs_rollback(true); - return (true); + return (false); } else testutil_die( ret, "unhandled error while trying to save an update to the tracking table"); } transaction.add_op(); - return (false); + return (true); +} + +bool +thread_context::insert( + scoped_cursor &cursor, uint64_t collection_id, uint64_t key_id, wt_timestamp_t ts) +{ + return insert(cursor, collection_id, key_to_string(key_id), ts); } bool -thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t key_id) +thread_context::insert( + scoped_cursor &cursor, uint64_t collection_id, const std::string &key, wt_timestamp_t ts) { WT_DECL_RET; - std::string key, value; + std::string value; testutil_assert(tracking != nullptr); testutil_assert(cursor.get() != nullptr); /* - * Get a timestamp to apply to the update. We still do this even if timestamps aren't enabled as - * it will return a value for the tracking table. + * When no timestamp is specified, get one to apply to the update. We still do this even if the + * timestamp manager is not enabled as it will return a value for the tracking table. */ - wt_timestamp_t ts = tsm->get_next_ts(); + if (ts == 0) + ts = tsm->get_next_ts(); transaction.set_commit_timestamp(ts); - key = key_to_string(key_id); value = random_generator::instance().generate_pseudo_random_string(value_size); cursor->set_key(cursor.get(), key.c_str()); @@ -242,7 +252,7 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k if (ret != 0) { if (ret == WT_ROLLBACK) { transaction.set_needs_rollback(true); - return (true); + return (false); } else testutil_die(ret, "unhandled error while trying to insert a key"); } @@ -251,13 +261,13 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k if (ret != 0) { if (ret == WT_ROLLBACK) { transaction.set_needs_rollback(true); - return (true); + return (false); } else testutil_die( ret, "unhandled error while trying to save an insert to the tracking table"); } transaction.add_op(); - return (false); + return (true); } void diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 7bb6ac66cb6..61e1b99f28a 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -72,7 +72,7 @@ class transaction_context { /* Begin a transaction if we are not currently in one. */ void try_begin(const std::string &config = ""); /* - * Commit a transaction and return true if a rollback is required. + * Commit a transaction and return true if the commit was successful. */ bool commit(const std::string &config = ""); /* Rollback a transaction, failure will abort the test. */ @@ -136,16 +136,22 @@ class thread_context { /* * Generic update function, takes a collection_id and key, will generate the value. * - * Returns true if a rollback is required. + * Return true if the operation was successful, a return value of false implies the transaction + * needs to be rolled back. */ bool update(scoped_cursor &cursor, uint64_t collection_id, const std::string &key); /* - * Generic insert function, takes a collection_id and key_id, will generate the value. + * Generic insert function, takes a collection_id and key_id, will generate the value. If a + * timestamp is not specified, the timestamp manager will generate one. * - * Returns true if a rollback is required. + * Return true if the operation was successful, a return value of false implies the transaction + * needs to be rolled back. */ - bool insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t key_id); + bool insert( + scoped_cursor &cursor, uint64_t collection_id, uint64_t key_id, wt_timestamp_t ts = 0); + bool insert( + scoped_cursor &cursor, uint64_t collection_id, const std::string &key, wt_timestamp_t ts = 0); void sleep(); bool running() const; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cxx new file mode 100644 index 00000000000..9d5318e7acb --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/burst_inserts.cxx @@ -0,0 +1,162 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/test.h" +#include "test_harness/workload/random_generator.h" +#include "test_harness/timestamp_manager.h" + +using namespace test_harness; + +/* + * This test inserts and reads a large quantity of data in bursts, this is intended to simulate a + * mongod instance loading a large amount of data over a long period of time. + */ +class burst_inserts : public test { + public: + burst_inserts(const test_args &args) : test(args) + { + _burst_duration = _config->get_int("burst_duration"); + logger::log_msg(LOG_INFO, "Burst duration set to: " + std::to_string(_burst_duration)); + } + + /* + * Insert operation that inserts continuously for insert_duration with no throttling. It then + * sleeps for op_rate. + */ + void + insert_operation(thread_context *tc) override final + { + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + /* Helper struct which stores a pointer to a collection and a cursor associated with it. */ + struct collection_cursor { + collection_cursor( + collection &coll, scoped_cursor &&write_cursor, scoped_cursor &&read_cursor) + : coll(coll), write_cursor(std::move(write_cursor)), + read_cursor(std::move(read_cursor)) + { + } + collection &coll; + scoped_cursor read_cursor; + scoped_cursor write_cursor; + }; + + /* Collection cursor vector. */ + std::vector<collection_cursor> ccv; + uint64_t collection_count = tc->db.get_collection_count(); + uint64_t collections_per_thread = collection_count / tc->thread_count; + /* Must have unique collections for each thread. */ + testutil_assert(collection_count % tc->thread_count == 0); + int thread_offset = tc->id * collections_per_thread; + for (int i = thread_offset; i < thread_offset + collections_per_thread && tc->running(); + ++i) { + collection &coll = tc->db.get_collection(i); + /* + * Create a reading cursor that will read random documents for every next call. This + * will help generate cache pressure. + */ + ccv.push_back({coll, std::move(tc->session.open_scoped_cursor(coll.name.c_str())), + std::move(tc->session.open_scoped_cursor(coll.name.c_str(), "next_random=true"))}); + } + + uint64_t counter = 0; + while (tc->running()) { + uint64_t start_key = ccv[counter].coll.get_key_count(); + uint64_t added_count = 0; + bool committed = true; + auto &cc = ccv[counter]; + auto burst_start = std::chrono::system_clock::now(); + while (tc->running() && + std::chrono::system_clock::now() - burst_start < + std::chrono::seconds(_burst_duration)) { + tc->transaction.try_begin(); + cc.write_cursor->set_key( + cc.write_cursor.get(), tc->key_to_string(start_key + added_count).c_str()); + cc.write_cursor->search(cc.write_cursor.get()); + + /* A return value of true implies the insert was successful. */ + if (!tc->insert(cc.write_cursor, cc.coll.id, start_key + added_count)) { + tc->transaction.rollback(); + added_count = 0; + continue; + } + added_count++; + + /* Walk our random reader intended to generate cache pressure. */ + int ret = 0; + if ((ret = cc.read_cursor->next(cc.read_cursor.get())) != 0) { + if (ret == WT_NOTFOUND) { + cc.read_cursor->reset(cc.read_cursor.get()); + } else if (ret == WT_ROLLBACK) { + tc->transaction.rollback(); + added_count = 0; + continue; + } else { + testutil_die(ret, "Unhandled error in cursor->next()"); + } + } + + if (tc->transaction.can_commit()) { + if (tc->transaction.commit()) { + cc.coll.increase_key_count(added_count); + start_key = cc.coll.get_key_count(); + } + added_count = 0; + } + + /* Sleep as currently this loop is too fast. */ + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + /* Close out our current txn. */ + if (tc->transaction.active()) { + if (tc->transaction.commit()) { + logger::log_msg(LOG_TRACE, + "Committed an insertion of " + std::to_string(added_count) + " keys."); + cc.coll.increase_key_count(added_count); + start_key = cc.coll.get_key_count(); + } + added_count = 0; + } + + testutil_check(cc.write_cursor->reset(cc.write_cursor.get())); + testutil_check(cc.read_cursor->reset(cc.read_cursor.get())); + counter++; + if (counter == collections_per_thread) + counter = 0; + testutil_assert(counter < collections_per_thread); + tc->sleep(); + } + /* Make sure the last transaction is rolled back now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + } + + private: + int _burst_duration = 0; +}; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx index a226fa33ff3..5a1996b45a1 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx @@ -37,21 +37,26 @@ class example_test : public test_harness::test { example_test(const test_harness::test_args &args) : test(args) {} void - populate(test_harness::database &database, test_harness::timestamp_manager *_timestamp_manager, - test_harness::configuration *_config, - test_harness::workload_tracking *tracking) override final + populate(test_harness::database &, test_harness::timestamp_manager *, + test_harness::configuration *, test_harness::workload_tracking *) override final { std::cout << "populate: nothing done." << std::endl; } void - read_operation(test_harness::thread_context *context) override final + insert_operation(test_harness::thread_context *) override final + { + std::cout << "insert_operation: nothing done." << std::endl; + } + + void + read_operation(test_harness::thread_context *) override final { std::cout << "read_operation: nothing done." << std::endl; } void - update_operation(test_harness::thread_context *context) override final + update_operation(test_harness::thread_context *) override final { std::cout << "update_operation: nothing done." << std::endl; } diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx index 03de6947a86..5a828f865f5 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx @@ -91,14 +91,13 @@ class hs_cleanup : public test { * API doesn't guarantee our buffer will still be valid once it is called, as such we * copy the buffer and then pass it into the API. */ - bool rollback_required = tc->update(cursor, coll.id, key_value_t(key_tmp)); - - /* Commit our transaction. */ - if (!rollback_required && tc->transaction.can_commit()) - rollback_required = tc->transaction.commit(); - - if (rollback_required) + if (tc->update(cursor, coll.id, key_value_t(key_tmp))) { + if (tc->transaction.can_commit()) { + tc->transaction.commit(); + } + } else { tc->transaction.rollback(); + } } /* Ensure our last transaction is resolved. */ if (tc->transaction.active()) diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx index 4e436ff7af3..0c231e6568d 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx @@ -35,6 +35,7 @@ #include "base_test.cxx" #include "example_test.cxx" +#include "burst_inserts.cxx" #include "hs_cleanup.cxx" std::string @@ -114,6 +115,8 @@ run_test(const std::string &test_name, const std::string &config, const std::str example_test(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "hs_cleanup") hs_cleanup(test_harness::test_args{config, test_name, wt_open_config}).run(); + else if (test_name == "burst_inserts") + burst_inserts(test_harness::test_args{config, test_name, wt_open_config}).run(); else { test_harness::logger::log_msg(LOG_ERROR, "Test not found: " + test_name); error_code = -1; @@ -136,7 +139,8 @@ main(int argc, char *argv[]) { std::string cfg, config_filename, current_cfg, current_test_name, test_name, wt_open_config; int64_t error_code = 0; - const std::vector<std::string> all_tests = {"example_test", "hs_cleanup", "base_test"}; + const std::vector<std::string> all_tests = { + "example_test", "burst_inserts", "hs_cleanup", "base_test"}; /* Set the program name for error messages. */ (void)testutil_set_progname(argv); diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index f0935ba2c05..2e7d5f93480 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -31,6 +31,10 @@ test_scope_SOURCES = scope/main.c noinst_PROGRAMS += test_scope all_TESTS += test_scope +test_tiered_abort_SOURCES = tiered_abort/main.c +noinst_PROGRAMS += test_tiered_abort +all_TESTS += tiered_abort/smoke.sh + test_timestamp_abort_SOURCES = timestamp_abort/main.c noinst_PROGRAMS += test_timestamp_abort all_TESTS += timestamp_abort/smoke.sh @@ -151,6 +155,14 @@ test_wt6616_checkpoint_oldest_ts_SOURCES = wt6616_checkpoint_oldest_ts/main.c noinst_PROGRAMS += test_wt6616_checkpoint_oldest_ts all_TESTS += wt6616_checkpoint_oldest_ts/smoke.sh +test_wt7989_compact_checkpoint_SOURCES = wt7989_compact_checkpoint/main.c +noinst_PROGRAMS += test_wt7989_compact_checkpoint +all_TESTS += test_wt7989_compact_checkpoint + +test_wt8057_compact_stress_SOURCES = wt8057_compact_stress/main.c +noinst_PROGRAMS += test_wt8057_compact_stress +all_TESTS += test_wt8057_compact_stress + # Run this during a "make check" smoke test. TESTS = $(all_TESTS) LOG_COMPILER = env top_builddir=$(top_builddir) top_srcdir=$(top_srcdir) $(TEST_WRAPPER) diff --git a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c new file mode 100644 index 00000000000..c29460b8f46 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c @@ -0,0 +1,906 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_util.h" + +#include <sys/wait.h> +#include <signal.h> + +static char home[1024]; /* Program working dir */ + +/* + * Create three tables that we will write the same data to and verify that + * all the types of usage have the expected data in them after a crash and + * recovery. We want: + * 1. A table that is logged and is not involved in timestamps. This table + * simulates a user local table. + * 2. A table that is logged and involved in timestamps. This simulates + * the oplog. + * 3. A table that is not logged and involved in timestamps. This simulates + * a typical collection file. We also insert identical data into a shadow table + * with a different timestamp that simulates insertion on a secondary. + * + * We also create another table that is not logged and not involved directly + * in timestamps to store the stable timestamp. That way we can know what the + * latest stable timestamp is on checkpoint. + * + * We also create several files that are not WiredTiger tables. The flush tier thread creates + * one such file indicating that the specified number of flush_tier calls have completed. The parent + * process uses this to know when that threshold is met and it can start the timer to abort. + * Also each worker thread creates its own textual records file that records the data it + * inserted and it records the timestamp that was used for that insertion. + */ +#define BUCKET "bucket" +#define INVALID_KEY UINT64_MAX +#define MAX_CKPT_INVL 5 /* Maximum interval between checkpoints */ +#define MAX_FLUSH_INVL 5 /* Maximum interval between flush_tier calls */ +#define MAX_TH 20 /* Maximum configurable threads */ +#define MAX_TIME 40 +#define MAX_VAL 1024 +#define MIN_TH 5 +#define MIN_TIME 10 +#define NUM_INT_THREADS 3 +#define RECORDS_FILE "records-%" PRIu32 +/* Include worker threads and extra sessions */ +#define SESSION_MAX (MAX_TH + 4) +#define WT_STORAGE_LIB "ext/storage_sources/local_store/.libs/libwiredtiger_local_store.so" + +static const char *table_pfx = "table"; +static const char *const uri_collection = "collection"; +static const char *const uri_local = "local"; +static const char *const uri_oplog = "oplog"; +static const char *const uri_shadow = "shadow"; + +static const char *const sentinel_file = "sentinel_ready"; + +static bool use_ts; +static volatile uint64_t global_ts = 1; +static uint32_t flush_calls = 1; + +/* + * The configuration sets the eviction update and dirty targets at 20% so that on average, each + * thread can have a couple of dirty pages before eviction threads kick in. See below where these + * symbols are used for cache sizing - we'll have about 10 pages allocated per thread. On the other + * side, the eviction update and dirty triggers are 90%, so application threads aren't involved in + * eviction until we're close to running out of cache. + */ +#define ENV_CONFIG_DEF \ + "cache_size=%" PRIu32 \ + "M,create," \ + "debug_mode=(table_logging=true,checkpoint_retention=5)," \ + "eviction_updates_target=20,eviction_updates_trigger=90," \ + "log=(archive=true,file_max=10M,enabled),session_max=%d," \ + "statistics=(fast),statistics_log=(wait=1,json=true)," \ + "tiered_storage=(bucket=%s,bucket_prefix=pfx,name=local_store)" +#define ENV_CONFIG_TXNSYNC \ + ENV_CONFIG_DEF \ + ",eviction_dirty_target=20,eviction_dirty_trigger=90" \ + ",transaction_sync=(enabled,method=none)" +#define ENV_CONFIG_REC "log=(archive=false,recover=on)" + +/* + * A minimum width of 10, along with zero filling, means that all the keys sort according to their + * integer value, making each thread's key space distinct. + */ +#define KEY_FORMAT ("%010" PRIu64) + +typedef struct { + uint64_t absent_key; /* Last absent key */ + uint64_t exist_key; /* First existing key after miss */ + uint64_t first_key; /* First key in range */ + uint64_t first_miss; /* First missing key */ + uint64_t last_key; /* Last key in range */ +} REPORT; + +typedef struct { + WT_CONNECTION *conn; + uint64_t start; + uint32_t info; +} THREAD_DATA; + +/* + * TODO: WT-7833 Lock to coordinate inserts and flush_tier. This lock should be removed when that + * ticket is fixed. Flush_tier should be able to run with ongoing operations. + */ +static pthread_rwlock_t flush_lock; +/* Lock for transactional ops that set or query a timestamp. */ +static pthread_rwlock_t ts_lock; + +static void handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void +usage(void) +{ + fprintf(stderr, "usage: %s [-h dir] [-T threads] [-t time] [-vz]\n", progname); + exit(EXIT_FAILURE); +} + +/* + * thread_ts_run -- + * Runner function for a timestamp thread. + */ +static WT_THREAD_RET +thread_ts_run(void *arg) +{ + WT_DECL_RET; + WT_SESSION *session; + THREAD_DATA *td; + char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE]; + + td = (THREAD_DATA *)arg; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + /* Update the oldest timestamp every 1 millisecond. */ + for (;;) { + /* + * We get the last committed timestamp periodically in order to update the oldest timestamp, + * that requires locking out transactional ops that set or query a timestamp. + */ + testutil_check(pthread_rwlock_wrlock(&ts_lock)); + ret = td->conn->query_timestamp(td->conn, ts_string, "get=all_durable"); + testutil_check(pthread_rwlock_unlock(&ts_lock)); + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + if (ret == 0) { + /* + * Set both the oldest and stable timestamp so that we don't need to maintain read + * availability at older timestamps. + */ + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string)); + testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + } + __wt_sleep(0, 1000); + } + /* NOTREACHED */ +} + +/* + * thread_ckpt_run -- + * Runner function for the checkpoint thread. + */ +static WT_THREAD_RET +thread_ckpt_run(void *arg) +{ + WT_RAND_STATE rnd; + WT_SESSION *session; + THREAD_DATA *td; + uint64_t stable; + uint32_t sleep_time; + int i; + char ts_string[WT_TS_HEX_STRING_SIZE]; + + __wt_random_init(&rnd); + + td = (THREAD_DATA *)arg; + /* + * Keep a separate file with the records we wrote for checking. + */ + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + for (i = 0;; ++i) { + sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL; + sleep(sleep_time); + /* + * Since this is the default, send in this string even if running without timestamps. + */ + testutil_check(session->checkpoint(session, "use_timestamp=true")); + testutil_check(td->conn->query_timestamp(td->conn, ts_string, "get=last_checkpoint")); + testutil_assert(sscanf(ts_string, "%" SCNx64, &stable) == 1); + printf("Checkpoint %d complete at stable %" PRIu64 ".\n", i, stable); + fflush(stdout); + } + /* NOTREACHED */ +} + +/* + * thread_flush_run -- + * Runner function for the flush_tier thread. + */ +static WT_THREAD_RET +thread_flush_run(void *arg) +{ + FILE *fp; + WT_RAND_STATE rnd; + WT_SESSION *session; + THREAD_DATA *td; + uint64_t stable; + uint32_t i, sleep_time; + char ts_string[WT_TS_HEX_STRING_SIZE]; + + __wt_random_init(&rnd); + + td = (THREAD_DATA *)arg; + /* + * Keep a separate file with the records we wrote for checking. + */ + (void)unlink(sentinel_file); + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + for (i = 0;; ++i) { + sleep_time = __wt_random(&rnd) % MAX_FLUSH_INVL; + sleep(sleep_time); + testutil_check(td->conn->query_timestamp(td->conn, ts_string, "get=last_checkpoint")); + testutil_assert(sscanf(ts_string, "%" SCNx64, &stable) == 1); + /* Effectively wait for the first checkpoint to complete. */ + if (use_ts && stable == WT_TS_NONE) + continue; + /* + * Currently not testing any of the flush tier configuration strings other than defaults. We + * expect the defaults are what MongoDB wants for now. + */ + testutil_check(pthread_rwlock_wrlock(&flush_lock)); + testutil_check(session->flush_tier(session, NULL)); + testutil_check(pthread_rwlock_unlock(&flush_lock)); + printf("Flush tier %" PRIu32 " completed.\n", i); + fflush(stdout); + /* + * Create the sentinel file so that the parent process knows the desired number of + * flush_tier calls have finished and can start its timer. + */ + if (i == flush_calls) { + testutil_checksys((fp = fopen(sentinel_file, "w")) == NULL); + testutil_checksys(fclose(fp) != 0); + } + } + /* NOTREACHED */ +} + +/* + * thread_run -- + * Runner function for the worker threads. + */ +static WT_THREAD_RET +thread_run(void *arg) +{ + FILE *fp; + WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_shadow; + WT_DECL_RET; + WT_ITEM data; + WT_RAND_STATE rnd; + WT_SESSION *session; + THREAD_DATA *td; + uint64_t i, active_ts; + char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL]; + char kname[64], tscfg[64], uri[128]; + bool locked; + + __wt_random_init(&rnd); + memset(cbuf, 0, sizeof(cbuf)); + memset(lbuf, 0, sizeof(lbuf)); + memset(obuf, 0, sizeof(obuf)); + memset(kname, 0, sizeof(kname)); + locked = false; + + td = (THREAD_DATA *)arg; + /* + * Set up the separate file for checking. + */ + testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), RECORDS_FILE, td->info)); + (void)unlink(cbuf); + testutil_checksys((fp = fopen(cbuf, "w")) == NULL); + /* + * Set to line buffering. But that is advisory only. We've seen cases where the result files end + * up with partial lines. + */ + __wt_stream_set_line_buffer(fp); + + testutil_check(td->conn->open_session(td->conn, NULL, "isolation=snapshot", &session)); + /* + * Open a cursor to each table. + */ + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cur_coll)); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_shadow)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cur_shadow)); + + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cur_local)); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cur_oplog)); + + /* + * Write our portion of the key space until we're killed. + */ + printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->info, td->start); + active_ts = 0; + for (i = td->start;; ++i) { + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, i)); + + testutil_check(session->begin_transaction(session, NULL)); + + if (use_ts) { + testutil_check(pthread_rwlock_rdlock(&ts_lock)); + active_ts = __wt_atomic_addv64(&global_ts, 2); + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts)); + /* + * Set the transaction's timestamp now before performing the operation. + */ + testutil_check(session->timestamp_transaction(session, tscfg)); + testutil_check(pthread_rwlock_unlock(&ts_lock)); + } + + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + cur_shadow->set_key(cur_shadow, kname); + /* + * Put an informative string into the value so that it can be viewed well in a binary dump. + */ + testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), + "COLL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, active_ts, i)); + testutil_check(__wt_snprintf(lbuf, sizeof(lbuf), + "LOCAL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, active_ts, i)); + testutil_check(__wt_snprintf(obuf, sizeof(obuf), + "OPLOG: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, active_ts, i)); + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = cbuf; + cur_coll->set_value(cur_coll, &data); + testutil_check(pthread_rwlock_rdlock(&flush_lock)); + locked = true; + if ((ret = cur_coll->insert(cur_coll)) == WT_ROLLBACK) + goto rollback; + testutil_check(ret); + cur_shadow->set_value(cur_shadow, &data); + if (use_ts) { + /* + * Change the timestamp in the middle of the transaction so that we simulate a + * secondary. + */ + ++active_ts; + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts)); + testutil_check(session->timestamp_transaction(session, tscfg)); + } + if ((ret = cur_shadow->insert(cur_shadow)) == WT_ROLLBACK) + goto rollback; + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = obuf; + cur_oplog->set_value(cur_oplog, &data); + if ((ret = cur_oplog->insert(cur_oplog)) == WT_ROLLBACK) + goto rollback; + testutil_check(session->commit_transaction(session, NULL)); + /* + * Insert into the local table outside the timestamp txn. This must occur after the + * timestamp transaction, not before, because of the possibility of rollback in the + * transaction. The local table must stay in sync with the other tables. + */ + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = lbuf; + cur_local->set_value(cur_local, &data); + testutil_check(cur_local->insert(cur_local)); + testutil_check(pthread_rwlock_unlock(&flush_lock)); + locked = false; + + /* Save the timestamps and key separately for checking later. */ + if (fprintf(fp, "%" PRIu64 " %" PRIu64 " %" PRIu64 "\n", active_ts, active_ts, i) < 0) + testutil_die(EIO, "fprintf"); + + if (0) { +rollback: + testutil_check(session->rollback_transaction(session, NULL)); + if (locked) { + testutil_check(pthread_rwlock_unlock(&flush_lock)); + locked = false; + } + } + } + /* NOTREACHED */ +} + +/* + * Child process creates the database and table, and then creates worker threads to add data until + * it is killed by the parent. + */ +static void run_workload(uint32_t, const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void +run_workload(uint32_t nth, const char *build_dir) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + THREAD_DATA *td; + wt_thread_t *thr; + uint32_t cache_mb, ckpt_id, flush_id, i, ts_id; + char envconf[1024], extconf[512], uri[128]; + + thr = dcalloc(nth + NUM_INT_THREADS, sizeof(*thr)); + td = dcalloc(nth + NUM_INT_THREADS, sizeof(THREAD_DATA)); + + /* + * Size the cache appropriately for the number of threads. Each thread adds keys sequentially to + * its own portion of the key space, so each thread will be dirtying one page at a time. By + * default, a leaf page grows to 32K in size before it splits and the thread begins to fill + * another page. We'll budget for 10 full size leaf pages per thread in the cache plus a little + * extra in the total for overhead. + */ + cache_mb = ((32 * WT_KILOBYTE * 10) * nth) / WT_MEGABYTE + 20; + + if (chdir(home) != 0) + testutil_die(errno, "Child chdir: %s", home); + testutil_check( + __wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_TXNSYNC, cache_mb, SESSION_MAX, BUCKET)); + + testutil_check(__wt_snprintf(extconf, sizeof(extconf), ",extensions=(%s/%s=(early_load=true))", + build_dir, WT_STORAGE_LIB)); + + strcat(envconf, extconf); + printf("wiredtiger_open configuration: %s\n", envconf); + testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * Create all the tables. + */ + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); + testutil_check( + session->create(session, uri, "key_format=S,value_format=u,log=(enabled=false)")); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_shadow)); + testutil_check( + session->create(session, uri, "key_format=S,value_format=u,log=(enabled=false)")); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); + testutil_check(session->create(session, uri, "key_format=S,value_format=u")); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); + testutil_check(session->create(session, uri, "key_format=S,value_format=u")); + /* + * Don't log the stable timestamp table so that we know what timestamp was stored at the + * checkpoint. + */ + testutil_check(session->close(session, NULL)); + + /* + * The checkpoint thread and the timestamp threads are added at the end of the array. + */ + ckpt_id = nth; + td[ckpt_id].conn = conn; + td[ckpt_id].info = nth; + printf("Create checkpoint thread\n"); + testutil_check(__wt_thread_create(NULL, &thr[ckpt_id], thread_ckpt_run, &td[ckpt_id])); + flush_id = nth + 1; + td[flush_id].conn = conn; + td[flush_id].info = nth; + printf("Create flush thread\n"); + testutil_check(__wt_thread_create(NULL, &thr[flush_id], thread_flush_run, &td[flush_id])); + ts_id = nth + 2; + if (use_ts) { + td[ts_id].conn = conn; + td[ts_id].info = nth; + printf("Create timestamp thread\n"); + testutil_check(__wt_thread_create(NULL, &thr[ts_id], thread_ts_run, &td[ts_id])); + } + printf("Create %" PRIu32 " writer threads\n", nth); + printf("Create %" PRIu32 " writer threads\n", nth); + for (i = 0; i < nth; ++i) { + td[i].conn = conn; + td[i].start = WT_BILLION * (uint64_t)i; + td[i].info = i; + testutil_check(__wt_thread_create(NULL, &thr[i], thread_run, &td[i])); + } + /* + * The threads never exit, so the child will just wait here until it is killed. + */ + fflush(stdout); + for (i = 0; i <= ts_id; ++i) + testutil_check(__wt_thread_join(NULL, &thr[i])); + /* + * NOTREACHED + */ + free(thr); + free(td); + exit(EXIT_SUCCESS); +} + +extern int __wt_optind; +extern char *__wt_optarg; + +/* + * Initialize a report structure. Since zero is a valid key we cannot just clear it. + */ +static void +initialize_rep(REPORT *r) +{ + r->first_key = r->first_miss = INVALID_KEY; + r->absent_key = r->exist_key = r->last_key = INVALID_KEY; +} + +/* + * Print out information if we detect missing records in the middle of the data of a report + * structure. + */ +static void +print_missing(REPORT *r, const char *fname, const char *msg) +{ + if (r->exist_key != INVALID_KEY) + printf("%s: %s error %" PRIu64 " absent records %" PRIu64 "-%" PRIu64 ". Then keys %" PRIu64 + "-%" PRIu64 " exist. Key range %" PRIu64 "-%" PRIu64 "\n", + fname, msg, (r->exist_key - r->first_miss) - 1, r->first_miss, r->exist_key - 1, + r->exist_key, r->last_key, r->first_key, r->last_key); +} + +/* + * Signal handler to catch if the child died unexpectedly. + */ +static void +handler(int sig) +{ + pid_t pid; + + WT_UNUSED(sig); + pid = wait(NULL); + /* The core file will indicate why the child exited. Choose EINVAL here. */ + testutil_die(EINVAL, "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); +} + +int +main(int argc, char *argv[]) +{ + struct sigaction sa; + struct stat sb; + FILE *fp; + REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH]; + TEST_OPTS *opts, _opts; + WT_CONNECTION *conn; + WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_shadow; + WT_RAND_STATE rnd; + WT_SESSION *session; + pid_t pid; + uint64_t absent_coll, absent_local, absent_oplog, absent_shadow, count, key, last_key; + uint64_t commit_fp, durable_fp, stable_val; + uint32_t i, nth, timeout; + int ch, status, ret; + const char *working_dir; + char buf[512], bucket_dir[512], build_dir[512], fname[64], kname[64]; + char envconf[1024], extconf[512]; + char ts_string[WT_TS_HEX_STRING_SIZE]; + bool fatal, rand_th, rand_time, verify_only; + + (void)testutil_set_progname(argv); + + use_ts = true; + nth = MIN_TH; + rand_th = rand_time = true; + timeout = MIN_TIME; + verify_only = false; + working_dir = "WT_TEST.tiered-abort"; + + while ((ch = __wt_getopt(progname, argc, argv, "f:h:T:t:vz")) != EOF) + switch (ch) { + case 'f': + flush_calls = (uint32_t)atoi(__wt_optarg); + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'T': + rand_th = false; + nth = (uint32_t)atoi(__wt_optarg); + if (nth > MAX_TH) { + fprintf( + stderr, "Number of threads is larger than the maximum %" PRId32 "\n", MAX_TH); + return (EXIT_FAILURE); + } + break; + case 't': + rand_time = false; + timeout = (uint32_t)atoi(__wt_optarg); + break; + case 'v': + verify_only = true; + break; + case 'z': + use_ts = false; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); + + /* + * Build the directory path needed for the extension after parsing the args. We are not using + * the opts variable other than for building the directory. We have already parsed the args + * we're interested in above. + */ + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_build_dir(opts, build_dir, 512); + + testutil_check(pthread_rwlock_init(&flush_lock, NULL)); + testutil_check(pthread_rwlock_init(&ts_lock, NULL)); + + testutil_work_dir_from_path(home, sizeof(home), working_dir); + /* + * If the user wants to verify they need to tell us how many threads there were so we can find + * the old record files. + */ + if (verify_only && rand_th) { + fprintf(stderr, "Verify option requires specifying number of threads\n"); + exit(EXIT_FAILURE); + } + if (!verify_only) { + /* Make both the home directory and the bucket directory under the home. */ + testutil_make_work_dir(home); + testutil_check(__wt_snprintf(bucket_dir, sizeof(bucket_dir), "%s/%s", working_dir, BUCKET)); + testutil_make_work_dir(bucket_dir); + + __wt_random_init_seed(NULL, &rnd); + if (rand_time) { + timeout = __wt_random(&rnd) % MAX_TIME; + if (timeout < MIN_TIME) + timeout = MIN_TIME; + } + if (rand_th) { + nth = __wt_random(&rnd) % MAX_TH; + if (nth < MIN_TH) + nth = MIN_TH; + } + + printf("Parent: timestamp in use: %s\n", use_ts ? "true" : "false"); + printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); + printf("CONFIG: %s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, !use_ts ? " -z" : "", + working_dir, nth, timeout); + /* + * Fork a child to insert as many items. We will then randomly kill the child, run recovery + * and make sure all items we wrote exist after recovery runs. + */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = handler; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + testutil_checksys((pid = fork()) < 0); + + if (pid == 0) { /* child */ + run_workload(nth, build_dir); + return (EXIT_SUCCESS); + } + + /* parent */ + /* + * Sleep for the configured amount of time before killing the child. Start the timeout from + * the time we notice that the file has been created. That allows the test to run correctly + * on really slow machines. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s", home, sentinel_file)); + while (stat(buf, &sb) != 0) + testutil_sleep_wait(1, pid); + sleep(timeout); + sa.sa_handler = SIG_DFL; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + + /* + * !!! It should be plenty long enough to make sure more than + * one log file exists. If wanted, that check would be added + * here. + */ + printf("Kill child\n"); + testutil_checksys(kill(pid, SIGKILL) != 0); + testutil_checksys(waitpid(pid, &status, 0) == -1); + } + + /* + * !!! If we wanted to take a copy of the directory before recovery, + * this is the place to do it. Don't do it all the time because + * it can use a lot of disk space, which can cause test machine + * issues. + */ + if (chdir(home) != 0) + testutil_die(errno, "parent chdir: %s", home); + + /* Copy the data to a separate folder for debugging purpose. */ + testutil_copy_data(home); + + printf("Open database, run recovery and verify content\n"); + + /* Open the connection which forces recovery to be run. */ + testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_REC)); + + testutil_check(__wt_snprintf(extconf, sizeof(extconf), ",extensions=(%s/%s=(early_load=true))", + build_dir, WT_STORAGE_LIB)); + + strcat(envconf, extconf); + testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* Open a cursor on all the tables. */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s:%s", table_pfx, uri_collection)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_coll)); + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s:%s", table_pfx, uri_shadow)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_shadow)); + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s:%s", table_pfx, uri_local)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_local)); + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_oplog)); + + /* Find the biggest stable timestamp value that was saved. */ + stable_val = 0; + if (use_ts) { + testutil_check(conn->query_timestamp(conn, ts_string, "get=recovery")); + testutil_assert(sscanf(ts_string, "%" SCNx64, &stable_val) == 1); + printf("Got stable_val %" PRIu64 "\n", stable_val); + } + + count = 0; + absent_coll = absent_local = absent_oplog = absent_shadow = 0; + fatal = false; + for (i = 0; i < nth; ++i) { + initialize_rep(&c_rep[i]); + initialize_rep(&l_rep[i]); + initialize_rep(&o_rep[i]); + testutil_check(__wt_snprintf(fname, sizeof(fname), RECORDS_FILE, i)); + if ((fp = fopen(fname, "r")) == NULL) + testutil_die(errno, "fopen: %s", fname); + + /* + * For every key in the saved file, verify that the key exists in the table after recovery. + * If we're doing in-memory log buffering we never expect a record missing in the middle, + * but records may be missing at the end. If we did write-no-sync, we expect every key to + * have been recovered. + */ + for (last_key = INVALID_KEY;; ++count, last_key = key) { + ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "%" SCNu64 "\n", &commit_fp, &durable_fp, &key); + if (last_key == INVALID_KEY) { + c_rep[i].first_key = key; + l_rep[i].first_key = key; + o_rep[i].first_key = key; + } + if (ret != EOF && ret != 3) { + /* If we find a partial line, consider it like an EOF. */ + if (ret == 2 || ret == 1 || ret == 0) + break; + testutil_die(errno, "fscanf"); + } + if (ret == EOF) + break; + /* + * If we're unlucky, the last line may be a partially written key at the end that can + * result in a false negative error for a missing record. Detect it. + */ + if (last_key != INVALID_KEY && key != last_key + 1) { + printf("%s: Ignore partial record %" PRIu64 " last valid key %" PRIu64 "\n", fname, + key, last_key); + break; + } + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, key)); + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + cur_shadow->set_key(cur_shadow, kname); + /* + * The collection table should always only have the data as of the checkpoint. The + * shadow table should always have the exact same data (or not) as the collection table, + * except for the last key that may be committed after the stable timestamp. + */ + if ((ret = cur_coll->search(cur_coll)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if ((ret = cur_shadow->search(cur_shadow)) == 0) + testutil_die(ret, "shadow search success"); + + /* + * If we don't find a record, the durable timestamp written to our file better be + * larger than the saved one. + */ + if (durable_fp != 0 && durable_fp <= stable_val) { + printf("%s: COLLECTION no record with key %" PRIu64 + " record durable ts %" PRIu64 " <= stable ts %" PRIu64 "\n", + fname, key, durable_fp, stable_val); + absent_coll++; + } + if (c_rep[i].first_miss == INVALID_KEY) + c_rep[i].first_miss = key; + c_rep[i].absent_key = key; + } else if ((ret = cur_shadow->search(cur_shadow)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "shadow search"); + /* + * We respectively insert the record to the collection table at timestamp t and to + * the shadow table at t + 1. If the checkpoint finishes at timestamp t, the last + * shadow table record will be removed by rollback to stable after restart. + */ + if (durable_fp <= stable_val) { + printf("%s: SHADOW no record with key %" PRIu64 "\n", fname, key); + absent_shadow++; + } + } else if (c_rep[i].absent_key != INVALID_KEY && c_rep[i].exist_key == INVALID_KEY) { + /* + * If we get here we found a record that exists after absent records, a hole in our + * data. + */ + c_rep[i].exist_key = key; + fatal = true; + } else if (commit_fp != 0 && commit_fp > stable_val) { + /* + * If we found a record, the commit timestamp written to our file better be no + * larger than the checkpoint one. + */ + printf("%s: COLLECTION record with key %" PRIu64 " commit record ts %" PRIu64 + " > stable ts %" PRIu64 "\n", + fname, key, commit_fp, stable_val); + fatal = true; + } else if ((ret = cur_shadow->search(cur_shadow)) != 0) + /* Collection and shadow both have the data. */ + testutil_die(ret, "shadow search failure"); + + /* The local table should always have all data. */ + if ((ret = cur_local->search(cur_local)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + printf("%s: LOCAL no record with key %" PRIu64 "\n", fname, key); + absent_local++; + if (l_rep[i].first_miss == INVALID_KEY) + l_rep[i].first_miss = key; + l_rep[i].absent_key = key; + } else if (l_rep[i].absent_key != INVALID_KEY && l_rep[i].exist_key == INVALID_KEY) { + /* We should never find an existing key after we have detected one missing. */ + l_rep[i].exist_key = key; + fatal = true; + } + /* The oplog table should always have all data. */ + if ((ret = cur_oplog->search(cur_oplog)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + printf("%s: OPLOG no record with key %" PRIu64 "\n", fname, key); + absent_oplog++; + if (o_rep[i].first_miss == INVALID_KEY) + o_rep[i].first_miss = key; + o_rep[i].absent_key = key; + } else if (o_rep[i].absent_key != INVALID_KEY && o_rep[i].exist_key == INVALID_KEY) { + /* We should never find an existing key after we have detected one missing. */ + o_rep[i].exist_key = key; + fatal = true; + } + } + c_rep[i].last_key = last_key; + l_rep[i].last_key = last_key; + o_rep[i].last_key = last_key; + testutil_checksys(fclose(fp) != 0); + print_missing(&c_rep[i], fname, "COLLECTION"); + print_missing(&l_rep[i], fname, "LOCAL"); + print_missing(&o_rep[i], fname, "OPLOG"); + } + testutil_check(conn->close(conn, NULL)); + if (absent_coll) { + printf("COLLECTION: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_coll, count); + fatal = true; + } + if (absent_shadow) { + printf("SHADOW: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_shadow, count); + fatal = true; + } + if (absent_local) { + printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_local, count); + fatal = true; + } + if (absent_oplog) { + printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_oplog, count); + fatal = true; + } + testutil_check(pthread_rwlock_destroy(&flush_lock)); + testutil_check(pthread_rwlock_destroy(&ts_lock)); + if (fatal) + return (EXIT_FAILURE); + printf("%" PRIu64 " records verified\n", count); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/csuite/tiered_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/tiered_abort/smoke.sh new file mode 100755 index 00000000000..b853c4fcd5e --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/tiered_abort/smoke.sh @@ -0,0 +1,24 @@ +#! /bin/sh + +set -e + +# Return success from the script because the test itself does not yet work. +# Do this in the script so that we can manually run the program on the command line. +exit 0 +# Smoke-test tiered-abort as part of running "make check". + +if [ -n "$1" ] +then + # If the test binary is passed in manually. + test_bin=$1 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_tiered_abort +fi +$TEST_WRAPPER $test_bin -t 10 -T 5 +$TEST_WRAPPER $test_bin -m -t 10 -T 5 +$TEST_WRAPPER $test_bin -C -t 10 -T 5 +$TEST_WRAPPER $test_bin -C -m -t 10 -T 5 diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index c9883800199..5f5c019d3ad 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -149,14 +149,11 @@ thread_ts_run(void *arg) WT_RAND_STATE rnd; WT_SESSION *session; THREAD_DATA *td; - wt_timestamp_t all_dur_ts, prev_all_dur_ts; uint32_t rand_op; int dbg; char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE]; bool first; - prev_all_dur_ts = WT_TS_NONE; - td = (THREAD_DATA *)arg; __wt_random_init(&rnd); @@ -172,15 +169,6 @@ thread_ts_run(void *arg) ret = td->conn->query_timestamp(td->conn, ts_string, "get=all_durable"); testutil_check(pthread_rwlock_unlock(&ts_lock)); testutil_assert(ret == 0 || ret == WT_NOTFOUND); - /* - * All durable can intermittently move backwards, we do not want to set stable and the - * oldest timestamps backwards - refer WT-8001. - */ - all_dur_ts = strtoul(ts_string, NULL, 16); - if (!first && all_dur_ts < prev_all_dur_ts) { - __wt_sleep(0, 1000); - continue; - } if (ret == 0) { rand_op = __wt_random(&rnd) % 4; /* @@ -204,7 +192,6 @@ thread_ts_run(void *arg) testutil_check(td->conn->set_timestamp(td->conn, tscfg)); } first = false; - prev_all_dur_ts = all_dur_ts; /* * Set and reset the checkpoint retention setting on a regular basis. We want to test * racing with the internal archive thread while we're here. @@ -446,7 +433,7 @@ thread_run(void *arg) durable_ahead_commit ? active_ts + 4 : active_ts)); /* Ensure the global timestamp is not behind the all durable timestamp. */ if (durable_ahead_commit) - __wt_atomic_addv64(&global_ts, 3); + __wt_atomic_addv64(&global_ts, 4); } else testutil_check( __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts)); diff --git a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c new file mode 100644 index 00000000000..6e388b54535 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c @@ -0,0 +1,316 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * This test executes two test cases: + * - One with WT_TIMING_STRESS_CHECKPOINT_SLOW flag. It adds 10 seconds sleep before each + * checkpoint. + * - Another test case synchronizes compact and checkpoint threads using a condition variable. + * The reason we have two tests here is that they give different output when configured + * with "verbose=[compact,compact_progress]". There's a chance these two cases are different. + */ + +#define NUM_RECORDS 1000000 + +/* Constants and variables declaration. */ +/* + * You may want to add "verbose=[compact,compact_progress]" to the connection config string to get + * better view on what is happening. + */ +static const char conn_config[] = "create,cache_size=2GB,statistics=(all)"; +static const char table_config[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=i,value_format=QQQS"; +static char data_str[1024] = ""; + +/* Structures definition. */ +struct thread_data { + WT_CONNECTION *conn; + const char *uri; + WT_CONDVAR *cond; +}; + +/* Forward declarations. */ +static void run_test(bool stress_test, const char *home, const char *uri); +static void *thread_func_compact(void *arg); +static void *thread_func_checkpoint(void *arg); +static void populate(WT_SESSION *session, const char *uri); +static void remove_records(WT_SESSION *session, const char *uri); +static uint64_t get_file_size(WT_SESSION *session, const char *uri); +static void set_timing_stress_checkpoint(WT_CONNECTION *conn); + +/* Methods implementation. */ +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + char home_cv[512]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + + /* + * First run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. + */ + run_test(true, opts->home, opts->uri); + + /* + * Now run test where compact and checkpoint threads are synchronized using condition variable. + */ + testutil_assert(sizeof(home_cv) > strlen(opts->home) + 3); + sprintf(home_cv, "%s.CV", opts->home); + run_test(false, home_cv, opts->uri); + + /* Cleanup */ + if (!opts->preserve) + testutil_clean_work_dir(home_cv); + + testutil_cleanup(opts); + + return (EXIT_SUCCESS); +} + +static void +run_test(bool stress_test, const char *home, const char *uri) +{ + struct thread_data td; + WT_CONNECTION *conn; + WT_SESSION *session; + pthread_t thread_checkpoint, thread_compact; + uint64_t file_sz_after, file_sz_before; + + testutil_make_work_dir(home); + testutil_check(wiredtiger_open(home, NULL, conn_config, &conn)); + + if (stress_test) { + /* + * Set WT_TIMING_STRESS_CHECKPOINT_SLOW flag. It adds 10 seconds sleep before each + * checkpoint. + */ + set_timing_stress_checkpoint(conn); + } + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + /* Create and populate table. Checkpoint the data after that. */ + testutil_check(session->create(session, uri, table_config)); + + populate(session, uri); + testutil_check(session->checkpoint(session, NULL)); + + /* + * Remove 1/3 of data from the middle of the key range to let compact relocate blocks from the + * end of the file. + */ + remove_records(session, uri); + + file_sz_before = get_file_size(session, uri); + + td.conn = conn; + td.uri = uri; + td.cond = NULL; + + /* Spawn checkpoint and compact threads. Order is important! */ + if (stress_test) { + testutil_check(pthread_create(&thread_compact, NULL, thread_func_compact, &td)); + testutil_check(pthread_create(&thread_checkpoint, NULL, thread_func_checkpoint, &td)); + } else { + /* Create and initialize conditional variable. */ + testutil_check(__wt_cond_alloc((WT_SESSION_IMPL *)session, "compact operation", &td.cond)); + + testutil_check(pthread_create(&thread_checkpoint, NULL, thread_func_checkpoint, &td)); + testutil_check(pthread_create(&thread_compact, NULL, thread_func_compact, &td)); + } + + /* Wait for the threads to finish the work. */ + (void)pthread_join(thread_checkpoint, NULL); + (void)pthread_join(thread_compact, NULL); + + file_sz_after = get_file_size(session, uri); + + /* Cleanup */ + if (!stress_test) { + __wt_cond_destroy((WT_SESSION_IMPL *)session, &td.cond); + td.cond = NULL; + } + + testutil_check(session->close(session, NULL)); + + /* Check if there's at least 10% compaction. */ + printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n", + file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024)); + + /* + * FIXME-WT-8055 At the moment the assert below is commented out to prevent evergreen from going + * red. Please enable the assert as soon as the underlying defect is fixed and compact does its + * job well. + */ + /*testutil_assert(file_sz_before * 0.9 > file_sz_after);*/ +} + +static void * +thread_func_compact(void *arg) +{ + struct thread_data *td; + WT_SESSION *session; + + td = (struct thread_data *)arg; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (td->cond != NULL) { + /* + * Make sure checkpoint thread is initialized and waiting for the signal. Sleep for one + * second. + */ + __wt_sleep(1, 0); + + /* Wake up the checkpoint thread. */ + printf("Sending the signal!\n"); + __wt_cond_signal((WT_SESSION_IMPL *)session, td->cond); + } + + /* Perform compact operation. */ + testutil_check(session->compact(session, td->uri, NULL)); + testutil_check(session->close(session, NULL)); + + return (NULL); +} + +static bool +wait_run_check(WT_SESSION_IMPL *session) +{ + (void)session; /* Unused */ + + /* + * Always return true to make sure __wt_cond_wait_signal does wait. This callback is required + * with waits longer that one second. + */ + return (true); +} + +static void * +thread_func_checkpoint(void *arg) +{ + struct thread_data *td; + WT_SESSION *session; + bool signalled; + + td = (struct thread_data *)arg; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (td->cond != NULL) { + printf("Waiting for the signal...\n"); + /* + * Wait for the signal and time out after 20 seconds. wait_run_check is required because the + * time out is longer that one second. + */ + __wt_cond_wait_signal( + (WT_SESSION_IMPL *)session, td->cond, 20 * WT_MILLION, wait_run_check, &signalled); + testutil_assert(signalled); + printf("Signal received!\n"); + } + + testutil_check(session->checkpoint(session, NULL)); + testutil_check(session->close(session, NULL)); + + return (NULL); +} + +static void +populate(WT_SESSION *session, const char *uri) +{ + WT_CURSOR *cursor; + time_t t; + uint64_t val; + int i, str_len; + + srand((u_int)time(&t)); + + str_len = sizeof(data_str) / sizeof(data_str[0]); + for (i = 0; i < str_len - 1; i++) + data_str[i] = 'a' + rand() % 26; + + data_str[str_len - 1] = '\0'; + + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + for (i = 0; i < NUM_RECORDS; i++) { + cursor->set_key(cursor, i); + val = (uint64_t)rand(); + cursor->set_value(cursor, val, val, val, data_str); + testutil_check(cursor->insert(cursor)); + } + + testutil_check(cursor->close(cursor)); +} + +static void +remove_records(WT_SESSION *session, const char *uri) +{ + WT_CURSOR *cursor; + int i; + + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + + /* Remove 1/3 of the records from the middle of the key range. */ + for (i = NUM_RECORDS / 3; i < (NUM_RECORDS * 2) / 3; i++) { + cursor->set_key(cursor, i); + testutil_check(cursor->remove(cursor)); + } + + testutil_check(cursor->close(cursor)); +} + +static uint64_t +get_file_size(WT_SESSION *session, const char *uri) +{ + WT_CURSOR *cur_stat; + uint64_t val; + char *descr, *str_val; + char stat_uri[128]; + + sprintf(stat_uri, "statistics:%s", uri); + testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_SIZE); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, &val)); + testutil_check(cur_stat->close(cur_stat)); + + return (val); +} + +static void +set_timing_stress_checkpoint(WT_CONNECTION *conn) +{ + WT_CONNECTION_IMPL *conn_impl; + + conn_impl = (WT_CONNECTION_IMPL *)conn; + conn_impl->timing_stress_flags |= WT_TIMING_STRESS_CHECKPOINT_SLOW; +} diff --git a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c new file mode 100644 index 00000000000..1a495ccf06f --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c @@ -0,0 +1,380 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +#include <sys/wait.h> +#include <signal.h> + +/* + * This test verifies that there are no data inconsistencies if compact operation is interrupted by + * an unclean shutdown. To achieve this, the main process spawns a child process. The child process + * performs certain operations on two identical tables. The parent process randomly kills the child + * process and verifies that the data across two tables match after restart. + */ + +#define NUM_RECORDS 100000 +#define TIMEOUT 40 + +/* Constants and variables declaration. */ +/* + * You may want to add "verbose=[compact,compact_progress]" to the connection config string to get + * better view on what is happening. + */ +static const char conn_config[] = "create,cache_size=2GB,statistics=(all)"; +static const char table_config[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=i,value_format=S"; +static char data_str[1024] = ""; + +static const char ckpt_file[] = "WT_TEST.compact-stress/checkpoint_done"; +static const char working_dir[] = "WT_TEST.compact-stress"; +static const char uri1[] = "table:compact1"; +static const char uri2[] = "table:compact2"; + +/* + * subtest_error_handler -- + * Error event handler. + */ +static int +subtest_error_handler( + WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message) +{ + (void)(handler); + (void)(session); + (void)(error); + fprintf(stderr, "%s", message); + return (0); +} + +static WT_EVENT_HANDLER event_handler = { + subtest_error_handler, NULL, /* Message handler */ + NULL, /* Progress handler */ + NULL /* Close handler */ +}; + +static void sig_handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + +/* Forward declarations. */ +static void run_test(const char *home); +static void populate(WT_SESSION *session, int start, int end); +static void remove_records(WT_SESSION *session, const char *uri, int start, int end); +static void verify_tables(WT_SESSION *session); +static int verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2); +static uint64_t get_file_size(WT_SESSION *session, const char *uri); + +/* + * Signal handler to catch if the child died unexpectedly. + */ +static void +sig_handler(int sig) +{ + pid_t pid; + + WT_UNUSED(sig); + pid = wait(NULL); + /* + * The core file will indicate why the child exited. Choose EINVAL here. + */ + testutil_die(EINVAL, "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); +} + +/* Methods implementation. */ +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CONNECTION *conn; + WT_SESSION *session; + + char home[1024]; + int status; + pid_t pid; + struct sigaction sa; + struct stat sb; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + + testutil_work_dir_from_path(home, sizeof(home), working_dir); + + printf("Work directory: %s\n", home); + testutil_make_work_dir(home); + + /* Fork a child to create tables and perform operations on them. */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sig_handler; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + testutil_checksys((pid = fork()) < 0); + + if (pid == 0) { /* child */ + + run_test(home); + /* + * We do not expect test to reach here. The child process should have been killed by the + * parent process. + */ + printf("Child finished processing...\n"); + return (EXIT_FAILURE); + } + + /* parent */ + /* + * Sleep for the configured amount of time before killing the child. Start the timeout from the + * time we notice that child process has written a checkpoint. That allows the test to run + * correctly on really slow machines. + */ + while (stat(ckpt_file, &sb) != 0) + testutil_sleep_wait(1, pid); + + /* Sleep for a while. Let the child process do some operations on the tables. */ + sleep(TIMEOUT); + sa.sa_handler = SIG_DFL; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + + /* Kill the child process. */ + printf("Kill child\n"); + testutil_checksys(kill(pid, SIGKILL) != 0); + testutil_checksys(waitpid(pid, &status, 0) == -1); + + /* Reopen the connection and verify that the tables match each other. */ + testutil_check(wiredtiger_open(home, &event_handler, conn_config, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + verify_tables(session); + + /* Clean-up. */ + testutil_check(session->close(session, NULL)); + testutil_check(conn->close(conn, NULL)); + if (!opts->preserve) + testutil_clean_work_dir(home); + testutil_cleanup(opts); + + return (EXIT_SUCCESS); +} + +static void +run_test(const char *home) +{ + + WT_CONNECTION *conn; + WT_SESSION *session; + FILE *fp; + + bool first_ckpt; + int i; + int key_range_start; + time_t t; + uint64_t file_sz_after, file_sz_before; + + first_ckpt = false; + srand((u_int)time(&t)); + + testutil_check(wiredtiger_open(home, &event_handler, conn_config, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + /* Create and populate table. Checkpoint the data after that. */ + testutil_check(session->create(session, uri1, table_config)); + testutil_check(session->create(session, uri2, table_config)); + + populate(session, 0, NUM_RECORDS); + + /* + * Although we are repeating the steps 40 times, we expect parent process will kill us way + * before than that. + */ + for (i = 0; i < 40; i++) { + + printf("Running Loop: %i\n", i + 1); + testutil_check(session->checkpoint(session, NULL)); + /* + * Create the checkpoint file so that the parent process knows at least one checkpoint has + * finished and can start its timer. + */ + if (!first_ckpt) { + testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); + testutil_checksys(fclose(fp) != 0); + first_ckpt = true; + } + + /* + * Remove 1/3 of data from the middle of the key range to let compact relocate blocks from + * the end of the file. + */ + key_range_start = rand() % (((NUM_RECORDS * 2) / 3) - 1); + remove_records(session, uri1, key_range_start, key_range_start + NUM_RECORDS / 3); + remove_records(session, uri2, key_range_start, key_range_start + NUM_RECORDS / 3); + + /* Only perform compaction on the first table. */ + file_sz_before = get_file_size(session, uri1); + + testutil_check(session->compact(session, uri1, NULL)); + + file_sz_after = get_file_size(session, uri1); + + /* Check if there's at least 10% compaction. */ + printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n", + file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024)); + + /* Put the deleted records back. */ + populate(session, key_range_start, key_range_start + NUM_RECORDS / 3); + + sleep(1); + } + + /* Clean-up. */ + testutil_check(session->close(session, NULL)); + testutil_check(conn->close(conn, NULL)); +} + +static void +populate(WT_SESSION *session, int start, int end) +{ + WT_CURSOR *cursor_1; + WT_CURSOR *cursor_2; + time_t t; + int i, str_len; + + srand((u_int)time(&t)); + + str_len = sizeof(data_str) / sizeof(data_str[0]); + for (i = 0; i < str_len - 1; i++) + data_str[i] = 'a' + rand() % 26; + + data_str[str_len - 1] = '\0'; + + testutil_check(session->open_cursor(session, uri1, NULL, NULL, &cursor_1)); + testutil_check(session->open_cursor(session, uri2, NULL, NULL, &cursor_2)); + for (i = start; i < end; i++) { + cursor_1->set_key(cursor_1, i); + cursor_1->set_value(cursor_1, data_str); + testutil_check(cursor_1->insert(cursor_1)); + + cursor_2->set_key(cursor_2, i); + cursor_2->set_value(cursor_2, data_str); + testutil_check(cursor_2->insert(cursor_2)); + } + + testutil_check(cursor_1->close(cursor_1)); + testutil_check(cursor_2->close(cursor_2)); +} + +static void +remove_records(WT_SESSION *session, const char *uri, int start, int end) +{ + WT_CURSOR *cursor; + int i; + + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + + for (i = start; i < end; i++) { + cursor->set_key(cursor, i); + testutil_check(cursor->remove(cursor)); + } + + testutil_check(cursor->close(cursor)); +} + +static int +verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2) +{ + WT_CURSOR *cursor_1; + WT_CURSOR *cursor_2; + + char *val_1; + char *val_2; + + int key; + int ret; + int total_keys; + + size_t val_1_size; + size_t val_2_size; + + testutil_check(session->open_cursor(session, table1, NULL, NULL, &cursor_1)); + testutil_check(session->open_cursor(session, table2, NULL, NULL, &cursor_2)); + + /* Run over all keys in first table and verify they are present in the second table. */ + total_keys = 0; + while ((ret = cursor_1->next(cursor_1)) == 0) { + cursor_1->get_key(cursor_1, &key); + cursor_2->set_key(cursor_2, key); + testutil_check(cursor_2->search(cursor_2)); + testutil_check(cursor_1->get_value(cursor_1, &val_1)); + testutil_check(cursor_2->get_value(cursor_2, &val_2)); + + val_1_size = strlen(val_1); + val_2_size = strlen(val_2); + + testutil_assert(val_1_size == val_2_size); + testutil_assert(memcmp(val_1, val_2, val_1_size) == 0); + + ++total_keys; + } + testutil_assert(ret == WT_NOTFOUND); + + testutil_check(cursor_1->close(cursor_1)); + testutil_check(cursor_2->close(cursor_2)); + + /* Return the number of keys verified. */ + return (total_keys); +} + +static void +verify_tables(WT_SESSION *session) +{ + int total_keys_1; + int total_keys_2; + + /* + * Run over all keys in first table and verify they are present in the second table. Repeat with + * all keys from the second table and verify that they are present in the first table; + */ + + total_keys_1 = verify_tables_helper(session, uri1, uri2); + total_keys_2 = verify_tables_helper(session, uri2, uri1); + + testutil_assert(total_keys_2 == total_keys_1); + printf("%i Keys verified from the two tables. \n", total_keys_1); +} + +static uint64_t +get_file_size(WT_SESSION *session, const char *uri) +{ + WT_CURSOR *cur_stat; + uint64_t val; + char *descr, *str_val; + char stat_uri[128]; + + sprintf(stat_uri, "statistics:%s", uri); + testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_SIZE); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, &val)); + testutil_check(cur_stat->close(cur_stat)); + + return (val); +} diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 5bb3fd50bc2..ba9f33806d5 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -216,22 +216,25 @@ functions: set -o errexit set -o verbose - if [[ "${branch_name}" != "develop" ]]; then - echo "We only run the documentation update task on the WiredTiger (develop) Evergreen project." - exit 0 + # Check if specific branches are provided to the function through the expansion variable + # defined in the documentation-update build variant. If none are specified, use the + # current branch. + if [ -z ${doc_update_branches} ]; then + branches=$(git rev-parse --abbrev-ref HEAD) + else + branches=${doc_update_branches} fi - # Go through each supported WiredTiger Github branch to build the docs. - # Please note the "branch" variable used here is a normal shell variable to store the name of - # a WiredTiger Github branch, while the "branch_name" used above is an Evergreen built-in variable - # to store the name of the branch being tested by the Evergreen project. - for branch in develop mongodb-5.0 mongodb-4.4 mongodb-4.2 mongodb-4.0 ; do - echo "[Debug] Checking out branch $branch ..." - git checkout $branch && git pull --ff-only && git reset --hard origin/$branch && git clean -fdqx + # Because of Evergreen's expansion syntax, this is used to process each branch separately. + IFS=, + for branch in $branches; do + + echo "Checking out branch $branch ..." + git checkout $branch sh reconf # Java API is removed in newer branches via WT-6675. - if [ "$branch" = "mongodb-4.2" -o "$branch" = "mongodb-4.0" ]; then + if [ $branch == "mongodb-4.2" ] || [ $branch == "mongodb-4.0" ]; then ../configure CFLAGS="-DMIGHT_NOT_RUN -Wno-error" --enable-java --enable-python --enable-strict (cd lang/python && make ../../../lang/python/wiredtiger_wrap.c) (cd lang/java && make ../../../lang/java/wiredtiger_wrap.c) @@ -240,15 +243,20 @@ functions: (cd lang/python && make ../../../lang/python/wiredtiger_wrap.c) fi - (cd ../dist && sh s_docs) - (cd .. && rm -rf docs-$branch && mv docs docs-$branch) + (cd ../dist && sh s_docs && echo "The documentation for $branch was successfully generated.") + # Save generated documentation + (cd .. && mv docs docs-$branch) done + "update wiredtiger docs": - command: shell.exec params: - working_dir: "wiredtiger" shell: bash script: | + # Use a single function to update the documentation of each supported WiredTiger branch. + # This is useful as not all branches have a dedicated Evergreen project. Furthermore, the + # documentation-update task is not triggered by every commit. We rely on the activity of + # the develop branch to update the documentation of all supported branches. set -o errexit set -o verbose @@ -260,37 +268,36 @@ functions: git clone git@github.com:wiredtiger/wiredtiger.github.com.git cd wiredtiger.github.com - # Go through each supported WiredTiger Github branch to stage the doc changes. - # Please note the "branch" variable used here is a normal shell variable to store the name of - # a WiredTiger Github branch, while the "branch_name" used above is an Evergreen built-in variable - # to store the name of the branch being tested by the Evergreen project. - for branch in mongodb-4.0 mongodb-4.2 mongodb-4.4 mongodb-5.0 develop ; do - echo "[Debug] Copying over doc directory for branch $branch ..." - rsync -avq ../docs-$branch/ $branch/ + # Branches to update are defined through an expansion variable. + branches=${doc_update_branches} - # Commit and push the changes (if any). - git add $branch > /dev/null - done + # Go through each branch to stage the doc changes. + IFS=, + for branch in $branches; do - git commit -m "Update auto-generated develop docs" \ - --author="svc-bot-doc-build <svc-wiredtiger-doc-build@10gen.com>" > /dev/null - # Use separate shell.exec with "silent" directive to avoid exposing credentail in task log. + # Synchronize the generated documentation with the current one. + echo "Synchronizing documentation for branch $branch ..." + rsync -avq ../wiredtiger/docs-$branch/ $branch/ --delete + + # Commit and push the changes if any. + if [[ $(git status "$branch" --porcelain) ]]; then + git add $branch + git commit -m "Update auto-generated docs for $branch" \ + --author="svc-bot-doc-build <svc-wiredtiger-doc-build@10gen.com>" + else + echo "No documentation changes for $branch." + fi + + done - command: shell.exec params: - working_dir: "wiredtiger" + working_dir: "wiredtiger.github.com" shell: bash silent: true script: | set -o errexit - set -o verbose + git push https://"${doc-update-github-token}"@github.com/wiredtiger/wiredtiger.github.com - if [[ "${branch_name}" != "develop" ]]; then - echo "We only run the documentation update task on the WiredTiger (develop) Evergreen project." - exit 0 - fi - - cd wiredtiger.github.com - git push https://${doc-update-github-token}@github.com/wiredtiger/wiredtiger.github.com "make check directory": command: shell.exec params: @@ -378,6 +385,14 @@ functions: set -o errexit set -o verbose ${test_env_vars|} ./test_random_abort ${random_abort_args|} 2>&1 + "tiered abort test": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + ${test_env_vars|} ./test_tiered_abort ${tiered_abort_args|} 2>&1 "timestamp abort test": command: shell.exec params: @@ -689,18 +704,6 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: CC=gcc-4.8 CXX=g++-4.8 ADD_CFLAGS="-ggdb -fPIC" - - func: "compile wiredtiger" - vars: - configure_env_vars: CC=gcc-5 CXX=g++-5 ADD_CFLAGS="-ggdb -fPIC" - - func: "compile wiredtiger" - vars: - configure_env_vars: CC=gcc-6 CXX=g++-6 ADD_CFLAGS="-ggdb -fPIC" - - func: "compile wiredtiger" - vars: - configure_env_vars: CC=gcc-6 CXX=g++-6 ADD_CFLAGS="-ggdb -fPIC" CPPFLAGS=-fvisibility=hidden - - func: "compile wiredtiger" - vars: configure_env_vars: CC=gcc-7 CXX=g++-7 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: @@ -715,15 +718,6 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - configure_env_vars: CC=clang-3.9 CXX=clang++-3.9 ADD_CFLAGS="-ggdb -fPIC" - - func: "compile wiredtiger" - vars: - configure_env_vars: CC=clang-4.0 CXX=clang++-4.0 ADD_CFLAGS="-ggdb -fPIC" - - func: "compile wiredtiger" - vars: - configure_env_vars: CC=clang-5.0 CXX=clang++-5.0 ADD_CFLAGS="-ggdb -fPIC" - - func: "compile wiredtiger" - vars: configure_env_vars: CC=clang-6.0 CXX=clang++-6.0 ADD_CFLAGS="-ggdb -fPIC" - func: "compile wiredtiger" vars: @@ -1065,6 +1059,21 @@ tasks: ${test_env_vars|} $(pwd)/../test/csuite/schema_abort/smoke.sh 2>&1 + - name: csuite-tiered-abort-test + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/../test/csuite/tiered_abort/smoke.sh 2>&1 + - name: csuite-timestamp-abort-test tags: ["pull_request"] depends_on: @@ -1554,6 +1563,35 @@ tasks: ${test_env_vars|} $(pwd)/test/csuite/test_wt6616_checkpoint_oldest_ts 2>&1 + - name: csuite-wt7989-compact-checkpoint-test + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/test/csuite/test_wt7989_compact_checkpoint 2>&1 + + - name: csuite-wt8057-compact-stress-test + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + script: | + set -o errexit + set -o verbose + ${test_env_vars|} $(pwd)/test/csuite/test_wt8057_compact_stress 2>&1 + # End of csuite test tasks # Start of Python unit test tasks @@ -2097,20 +2135,22 @@ tasks: vars: # Don't use diagnostic - this test looks for timing problems that are more likely to occur without it posix_configure_flags: --enable-strict - # Temporarily disable mix and column file type - # FIXME after WT-6608 - #- func: "checkpoint test" - # vars: - # checkpoint_args: -t m -n 1000000 -k 5000000 -C cache_size=100MB - #- func: "checkpoint test" - # vars: - # checkpoint_args: -t c -n 1000000 -k 5000000 -C cache_size=100MB + - func: "checkpoint test" + vars: + checkpoint_args: -t m -n 1000000 -k 5000000 -C cache_size=100MB + - func: "checkpoint test" + vars: + checkpoint_args: -t c -n 1000000 -k 5000000 -C cache_size=100MB - func: "checkpoint test" vars: checkpoint_args: -t r -n 1000000 -k 5000000 -C cache_size=100MB - name: coverage-report commands: + - command: timeout.update + params: + exec_timeout_secs: 36000 # 10 hrs + timeout_secs: 1800 # 30 mins - func: "get project" - func: "compile wiredtiger" vars: @@ -2122,43 +2162,55 @@ tasks: unit_test_args: -v 2 --long - func: "format test" vars: - extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=snappy logging=1 logging_compression=snappy logging_prealloc=1 + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=row compression=snappy logging=1 logging_compression=snappy logging_prealloc=1 - func: "format test" vars: - extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row alter=1 backups=1 compaction=1 data_extend=1 prepare=1 salvage=1 statistics=1 statistics_server=1 verify=1 + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=row alter=1 backups=1 compaction=1 data_extend=1 prepare=1 salvage=1 statistics=1 statistics_server=1 verify=1 - func: "format test" vars: - extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row firstfit=1 internal_key_truncation=1 + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=row firstfit=1 internal_key_truncation=1 - func: "format test" vars: - extra_args: leak_memory=0 mmap=1 file_type=row checkpoints=0 in_memory=1 reverse=1 truncate=1 + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=row checkpoints=0 in_memory=1 reverse=1 truncate=1 - func: "format test" vars: - extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=zlib huffman_value=1 + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=row compression=zlib huffman_value=1 - func: "format test" vars: - extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row data_source=lsm bloom=1 - # FIXME-WT-6669: temporarily disable column store test - # - func: "format test" - # vars: - # extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=var compression=snappy checksum=uncompressed dictionary=1 repeat_data_pct=10 + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=row data_source=lsm bloom=1 - func: "format test" vars: - extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=lz4 prefix_compression=1 leaf_page_max=9 internal_page_max=9 key_min=256 value_min=256 - # FIXME-WT-6669: temporarily disable column store test - # - func: "format test" - # vars: - # extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=var leaf_page_max=9 internal_page_max=9 value_min=256 - # FIXME-WT-6669: temporarily disable column store test + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=row compression=lz4 prefix_compression=1 leaf_page_max=9 internal_page_max=9 key_min=256 value_min=256 + - func: "format test" + vars: + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=var compression=snappy checksum=uncompressed dictionary=1 repeat_data_pct=10 + - func: "format test" + vars: + config: ../../../test/format/CONFIG.coverage + extra_args: file_type=var leaf_page_max=9 internal_page_max=9 value_min=256 + # FIXME-WT-8016: temporarily disable FLCS # - func: "format test" # vars: - # extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=fix + # config: ../../../test/format/CONFIG.coverage + # extra_args: file_type=fix - command: shell.exec params: + shell: bash working_dir: "wiredtiger/build_posix" script: | set -o errexit set -o verbose + virtualenv -p python3 venv + source venv/bin/activate + pip3 install gcovr mkdir -p ../coverage_report GCOV=/opt/mongodbtoolchain/v3/bin/gcov gcovr -r .. -f ../src -e '.*/bt_(debug|dump|misc|salvage|vrfy).*' -e '.*/(log|progress|verify_build|strerror|env_msg|err_file|cur_config|os_abort)\..*' -e '.*_stat\..*' -e 'bench' -e 'examples' -e 'test' -e 'ext' -e 'dist' -e 'tools' -j 4 --html-details --html-self-contained -o ../coverage_report/2_coverage_report.html - command: s3.put @@ -2369,6 +2421,118 @@ tasks: cp ../../../bench/wtperf/split_heavy.wtperf . ./wtperf -O ./split_heavy.wtperf -o verbose=2 + - name: data-validation-stress-test-checkpoint + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args}" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s1 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 1" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s2 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 2" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s3 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 3" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s4 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 4" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s5 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 5" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s6 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 6" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s7 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 7" WT_TEST.$t' 120 + - name: format-failure-configs-test depends_on: - name: compile @@ -2685,26 +2849,27 @@ tasks: buildvariants: -- name: ubuntu1804 - display_name: "! Ubuntu 18.04" +- name: ubuntu2004 + display_name: "! Ubuntu 20.04" run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(echo "`grep -c ^processor /proc/cpuinfo` * 2" | bc) posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make + data_validation_stress_test_args: -t r -m -W 3 -D -p -x -n 100000 -k 100000 -C cache_size=100MB tasks: - name: ".pull_request !.windows_only !.pull_request_compilers" - name: linux-directio - distros: ubuntu1804-build + distros: ubuntu2004-build - name: syscall-linux - name: configure-combinations - name: checkpoint-filetypes-test - name: unit-test-zstd - name: unit-test-long - distros: ubuntu1804-large + distros: ubuntu2004-large - name: unit-test-random-seed - name: spinlock-gcc-test - name: spinlock-pthread-adaptive-test @@ -2714,11 +2879,19 @@ buildvariants: - name: long-test - name: static-wt-build-test - name: format-failure-configs-test - -- name: ubuntu1804-cmake - display_name: "* Ubuntu 18.04 CMake" + - name: data-validation-stress-test-checkpoint + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s1 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s2 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s3 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s4 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s5 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s6 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s7 + +- name: ubuntu2004-cmake + display_name: "* Ubuntu 20.04 CMake" run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL @@ -2732,10 +2905,10 @@ buildvariants: - name: make-check-test - name: unit-test -- name: ubuntu1804-asan - display_name: "! Ubuntu 18.04 ASAN" +- name: ubuntu2004-asan + display_name: "! Ubuntu 20.04 ASAN" run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang @@ -2755,10 +2928,10 @@ buildvariants: - name: ".pull_request !.windows_only !.pull_request_compilers !.python" - name: examples-c-test -- name: ubuntu1804-msan - display_name: "! Ubuntu 18.04 MSAN" +- name: ubuntu2004-msan + display_name: "! Ubuntu 20.04 MSAN" run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: configure_env_vars: # We don't compile C++ for memory sanitized testing as it creates false positives. @@ -2786,10 +2959,10 @@ buildvariants: - name: format-stress-pull-request-test - name: make-check-test -- name: ubuntu1804-ubsan - display_name: "! Ubuntu 18.04 UBSAN" +- name: ubuntu2004-ubsan + display_name: "! Ubuntu 20.04 UBSAN" run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang @@ -2818,10 +2991,10 @@ buildvariants: - name: cppsuite-base-test-default - name: cppsuite-hs-cleanup-default -- name: ubuntu1804-compilers - display_name: "! Ubuntu 18.04 Compilers" +- name: ubuntu2004-compilers + display_name: "! Ubuntu 20.04 Compilers" run_on: - - ubuntu1804-wt-build + - ubuntu2004-wt-build expansions: posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-lz4 --enable-snappy --enable-zlib --enable-zstd --enable-python python_binary: '/opt/mongodbtoolchain/v3/bin/python3' @@ -2830,10 +3003,10 @@ buildvariants: tasks: - name: ".pull_request_compilers" -- name: ubuntu1804-stress-tests - display_name: Ubuntu 18.04 Stress tests +- name: ubuntu2004-stress-tests + display_name: Ubuntu 20.04 Stress tests run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make @@ -2855,7 +3028,7 @@ buildvariants: display_name: "Large scale tests" batchtime: 480 # 3 times a day run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make posix_configure_flags: @@ -2868,13 +3041,13 @@ buildvariants: tasks: - name: many-dhandle-stress-test - name: many-collection-test - distros: ubuntu1804-wt-large + distros: ubuntu2004-wt-large - name: cppsuite-stress-tests display_name: "Cppsuite Stress Tests" batchtime: 480 # 3 times a day run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: test_env_vars: LD_LIBRARY_PATH=$(pwd)/../../.libs PATH=/opt/mongodbtoolchain/v3/bin:$PATH make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make @@ -2890,25 +3063,26 @@ buildvariants: display_name: "~ Package" batchtime: 1440 # 1 day run_on: - - ubuntu1804-test + - ubuntu2004-test tasks: - name: package - name: documentation-update display_name: "~ Documentation update" - batchtime: 1440 # 1 day + batchtime: 10080 # 7 days run_on: - ubuntu2004-test - tasks: - - name: doc-update expansions: configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH + doc_update_branches: develop,mongodb-5.0,mongodb-4.4,mongodb-4.2,mongodb-4.0 + tasks: + - name: doc-update - name: linux-no-ftruncate display_name: Linux no ftruncate batchtime: 1440 # 1 day run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) @@ -2954,7 +3128,7 @@ buildvariants: display_name: "Code statistics" batchtime: 10080 # 7 days run_on: - - ubuntu1804-test + - ubuntu2004-test expansions: test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) tasks: @@ -2973,7 +3147,7 @@ buildvariants: - name: compatibility-tests display_name: Compatibility tests run_on: - - ubuntu1804-test + - ubuntu2004-test tasks: - name: compatibility-test-for-newer-releases - name: import-compatibility-test @@ -3043,7 +3217,7 @@ buildvariants: display_name: "~ Little-endian (x86)" run_on: - ubuntu1804-test - batchtime: 10080 # 7 days + batchtime: 4320 # 3 days expansions: python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) @@ -3060,7 +3234,7 @@ buildvariants: - enterprise run_on: - ubuntu1804-zseries-build - batchtime: 10080 # 7 days + batchtime: 4320 # 3 days expansions: python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) diff --git a/src/third_party/wiredtiger/test/format/CONFIG.coverage b/src/third_party/wiredtiger/test/format/CONFIG.coverage new file mode 100644 index 00000000000..61545a9d40a --- /dev/null +++ b/src/third_party/wiredtiger/test/format/CONFIG.coverage @@ -0,0 +1,9 @@ +# A reasonable configuration for coverage testing. +btree.huffman_value=0 +cache.minimum=20 +checkpoints=1 +leak_memory=0 +mmap=1 +runs.rows=1000000:5000000 +runs.threads=4:32 +runs.timer=15 diff --git a/src/third_party/wiredtiger/test/format/CONFIG.endian b/src/third_party/wiredtiger/test/format/CONFIG.endian index 791b24a4c11..9c5f9479248 100644 --- a/src/third_party/wiredtiger/test/format/CONFIG.endian +++ b/src/third_party/wiredtiger/test/format/CONFIG.endian @@ -4,4 +4,4 @@ logging.archive=0 logging=1 runs.timer=4 runs.rows=1000000 -runs.type=row-store +runs.type=row,var diff --git a/src/third_party/wiredtiger/test/format/CONFIG.stress b/src/third_party/wiredtiger/test/format/CONFIG.stress index e5ec1e5f754..a57a4f2df51 100644 --- a/src/third_party/wiredtiger/test/format/CONFIG.stress +++ b/src/third_party/wiredtiger/test/format/CONFIG.stress @@ -4,5 +4,5 @@ cache.minimum=20 runs.rows=1000000:5000000 runs.threads=4:32 runs.timer=6:30 -runs.type=row-store +runs.type=row,var runs=100 diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index e3c1afea11a..6cf7c795774 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -123,7 +123,6 @@ config_run(void) break; } } - config_map_file_type(g.c_file_type, &g.type); if (!config_is_perm("runs.source")) { config_single("runs.source=table", false); @@ -1329,14 +1328,58 @@ config_single(const char *s, bool perm) static void config_map_file_type(const char *s, u_int *vp) { - if (strcmp(s, "fix") == 0 || strcmp(s, "fixed-length column-store") == 0) + uint32_t v; + const char *arg; + bool fix, row, var; + + arg = s; + + /* Accumulate choices. */ + fix = row = var = false; + while (*s != '\0') { + if (WT_PREFIX_SKIP(s, "fixed-length column-store") || WT_PREFIX_SKIP(s, "fix")) + fix = true; + else if (WT_PREFIX_SKIP(s, "row-store") || WT_PREFIX_SKIP(s, "row")) + row = true; + else if (WT_PREFIX_SKIP(s, "variable-length column-store") || WT_PREFIX_SKIP(s, "var")) + var = true; + else + testutil_die(EINVAL, "illegal file type configuration: %s", arg); + + if (*s == ',') /* Allow, but don't require, comma-separators. */ + ++s; + } + if (!fix && !row && !var) + testutil_die(EINVAL, "illegal file type configuration: %s", arg); + + /* Check for a single configuration. */ + if (fix && !row && !var) { *vp = FIX; - else if (strcmp(s, "var") == 0 || strcmp(s, "variable-length column-store") == 0) - *vp = VAR; - else if (strcmp(s, "row") == 0 || strcmp(s, "row-store") == 0) + return; + } + if (!fix && row && !var) { *vp = ROW; + return; + } + if (!fix && !row && var) { + *vp = VAR; + return; + } + + /* + * Handle multiple configurations. + * + * Fixed-length column-store is 10% in all cases. + * + * Variable-length column-store is 90% vs. fixed, 30% vs. fixed and row, and 40% vs row. + */ + v = mmrand(NULL, 1, 10); + if (fix && v == 1) + *vp = FIX; + else if (var && (v < 5 || !row)) + *vp = VAR; else - testutil_die(EINVAL, "illegal file type configuration: %s", s); + *vp = ROW; } /* diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index c5456810e60..cfb4f78ba27 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -280,7 +280,7 @@ static CONFIG c[] = { {"runs.timer", "run time (minutes)", C_IGNORE, 0, 0, UINT_MAX, &g.c_timer, NULL}, - {"runs.type", "object type (fix | var | row)", C_IGNORE | C_STRING, 0, 0, 0, NULL, + {"runs.type", "object type (fix | row | var)", C_IGNORE | C_STRING, 0, 0, 0, NULL, &g.c_file_type}, {"runs.verify_failure_dump", "configure page dump on repeatable read error", C_IGNORE | C_BOOL, 0, diff --git a/src/third_party/wiredtiger/test/format/smoke.sh b/src/third_party/wiredtiger/test/format/smoke.sh index 256789d1a1f..151d1be37b3 100755 --- a/src/third_party/wiredtiger/test/format/smoke.sh +++ b/src/third_party/wiredtiger/test/format/smoke.sh @@ -5,15 +5,16 @@ set -e # Smoke-test format as part of running "make check". args="-1 -c . " args="$args btree.compression=none " +args="$args cache.minimum=40 " args="$args logging_compression=none" -args="$args runs.ops=50000 " -args="$args runs.rows=10000 " +args="$args runs.ops=500000 " +args="$args runs.rows=100000 " args="$args runs.source=table " args="$args runs.threads=4 " -# Temporarily disabled +# Temporarily disable LSM and FLCS. # $TEST_WRAPPER ./t $args runs.type=fix # $TEST_WRAPPER ./t $args runs.type=row runs.source=lsm -# $TEST_WRAPPER ./t $args runs.type=var $TEST_WRAPPER ./t $args runs.type=row +$TEST_WRAPPER ./t $args runs.type=var diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index c961d086516..42e6bb8c2ef 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -280,6 +280,7 @@ timestamp_once(WT_SESSION *session, bool allow_lag, bool final) testutil_timestamp_parse(buf, &all_durable); else { testutil_assert(ret == WT_NOTFOUND); + lock_writeunlock(session, &g.ts_lock); return; } diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py index 58ccc505416..52b9c6bbb6a 100755 --- a/src/third_party/wiredtiger/test/suite/hook_tiered.py +++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py @@ -195,14 +195,15 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): # Is this test one we should skip? def skip_test(self, test): # Skip any test that contains one of these strings as a substring - skip = ["backup", # Can't backup a tiered table - "cursor13_ckpt", # Checkpoint tests with cached cursors - "cursor13_drops", # Tests that require working drop implementation - "cursor13_dup", # More cursor cache tests - "cursor13_reopens", # More cursor cache tests - "lsm", # If the test name tells us it uses lsm ignore it - "test_config_json", # create replacement can't handle a json config string - "test_cursor_big", # Cursor caching verified with stats + skip = ["backup", # Can't backup a tiered table + "cursor13_ckpt", # Checkpoint tests with cached cursors + "cursor13_drops", # Tests that require working drop implementation + "cursor13_dup", # More cursor cache tests + "cursor13_reopens", # More cursor cache tests + "lsm", # If the test name tells us it uses lsm ignore it + "modify_smoke_recover", # Copying WT dir doesn't copy the bucket directory + "test_config_json", # create replacement can't handle a json config string + "test_cursor_big", # Cursor caching verified with stats "tiered"] for item in skip: if item in str(test): @@ -217,19 +218,19 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): def setup_hooks(self): orig_connection_close = self.Connection['close'] - self.Connection['close'] = (wthooks.HOOK_REPLACE, lambda s, config: + self.Connection['close'] = (wthooks.HOOK_REPLACE, lambda s, config=None: connection_close_replace(orig_connection_close, s, config)) orig_session_alter = self.Session['alter'] - self.Session['alter'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + self.Session['alter'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_alter_replace(orig_session_alter, s, uri, config)) orig_session_compact = self.Session['compact'] - self.Session['compact'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + self.Session['compact'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_compact_replace(orig_session_compact, s, uri, config)) orig_session_create = self.Session['create'] - self.Session['create'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + self.Session['create'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_create_replace(orig_session_create, s, uri, config)) orig_session_drop = self.Session['drop'] @@ -241,11 +242,11 @@ class TieredHookCreator(wthooks.WiredTigerHookCreator): session_rename_replace(orig_session_rename, s, uri, newuri, config)) orig_session_salvage = self.Session['salvage'] - self.Session['salvage'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + self.Session['salvage'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_salvage_replace(orig_session_salvage, s, uri, config)) orig_session_verify = self.Session['verify'] - self.Session['verify'] = (wthooks.HOOK_REPLACE, lambda s, uri, config: + self.Session['verify'] = (wthooks.HOOK_REPLACE, lambda s, uri, config=None: session_verify_replace(orig_session_verify, s, uri, config)) self.wiredtiger['wiredtiger_open'] = (wthooks.HOOK_ARGS, wiredtiger_open_tiered) diff --git a/src/third_party/wiredtiger/test/suite/test_alter05.py b/src/third_party/wiredtiger/test/suite/test_alter05.py new file mode 100644 index 00000000000..1105d0402f6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_alter05.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wiredtiger import stat + +# test_alter05.py +# Check the alter command succeeds even if the file is modified. +class test_alter05(wttest.WiredTigerTestCase): + conn_config = "statistics=(all)" + name = "alter05" + + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + + def verify_metadata(self, metastr): + c = self.session.open_cursor('metadata:', None, None) + + # We must find a file type entry for this object and its value + # should contain the provided file meta string. + c.set_key('file:' + self.name) + self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND) + value = c.get_value() + self.assertTrue(value.find(metastr) != -1) + + c.close() + + # Alter file to change the metadata and verify. + def test_alter05(self): + uri = "file:" + self.name + entries = 100 + create_params = 'key_format=i,value_format=i,' + + self.session.create(uri, create_params + "log=(enabled=true)") + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + # Verify the string in the metadata. + self.verify_metadata('log=(enabled=true)') + + # Put some data in table. + self.session.begin_transaction() + c = self.session.open_cursor(uri, None) + for k in range(entries): + c[k+1] = 1 + c.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) + + prev_alter_chceckpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint) + + # Alter the table and verify. + self.session.alter(uri, 'log=(enabled=false)') + self.verify_metadata('log=(enabled=false)') + + alter_chceckpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint) + self.assertEqual(prev_alter_chceckpoints + 1, alter_chceckpoints) + prev_alter_chceckpoints = alter_chceckpoints + + # Open a cursor, insert some data and try to alter with cursor open. + c2 = self.session.open_cursor(uri, None) + for k in range(entries): + c2[k+1] = 2 + + self.assertRaisesException(wiredtiger.WiredTigerError, + lambda: self.session.alter(uri, 'log=(enabled=true)')) + self.verify_metadata('log=(enabled=false)') + + alter_chceckpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint) + self.assertEqual(prev_alter_chceckpoints + 1, alter_chceckpoints) + + c2.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_compact03.py b/src/third_party/wiredtiger/test/suite/test_compact03.py new file mode 100644 index 00000000000..732bd39a597 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_compact03.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_compact03.py +# Test that compact doesn't reduce the file size when there are overflow values at the +# end of file. +# + +import time, wiredtiger, wttest +from wiredtiger import stat +from wtscenario import make_scenarios + +# Test compact behaviour with overflow values. +class test_compact03(wttest.WiredTigerTestCase): + + uri='table:test_compact03' + + fileConfig = [ + ('1KB', dict(fileConfig='allocation_size=1KB,leaf_page_max=1KB')), + ('4KB', dict(fileConfig='allocation_size=4KB,leaf_page_max=4KB')), + ] + + useTruncate = [ + ('no_truncate', dict(truncate=False)), + ('truncate', dict(truncate=True)) + ] + scenarios = make_scenarios(fileConfig, useTruncate) + + # Enable stats and use a cache size that can fit table in the memory. + conn_config = 'statistics=(all),cache_size=100MB' + + # We want to test how compaction interacts with overflow values. Test flow is as follows: + # + # 1. Populate a table with relatively small page size. + # 2. Checkpoint and get stats on the table to confirm the size. + # 3. Add few thousand overflow values. It is expected that these will be written at the end of + # file. + # 4. Perform checkpoint to ensure overflow values are written on disk. + # 5. Delete middle ~90% of the normal values in the table. + # 6. Perform checkpoint so compact can find something to work with. + # 7. Call compact. + # 8. Get stats on compacted table expecting that there will no change in size given we have + # overflow keys at the end of file. + # 9. Insert some normal values again. They will be written in the free extents in the middle + # of the file. Therefore, expect no increase in file size. + # + # We want to have around 20000 leaf pages. With minimum 1KB page allocation size, the table + # is expected to have at least 25 MByte worth of data. We can then experiment with deleting + # range of keys in middle to test how comapction works. + + normalValue = "abcde" * 10 + overflowValue = "abcde" * 1000 + nrecords = 400000 # To create ~25 MB table + expectedTableSize = 20 # Mbytes + nOverflowRecords = 5000 + + # Return the size of the file + def getSize(self): + # To allow this to work on systems without ftruncate, + # get the portion of the file allocated, via 'statistics=(all)', + # not the physical file size, via 'statistics=(size)'. + cstat = self.session.open_cursor( + 'statistics:' + self.uri, None, 'statistics=(all)') + sz = cstat[stat.dsrc.block_size][2] + cstat.close() + return sz + + # Create a table, add keys with both big and small values. + def test_compact03(self): + + mb = 1024 * 1024 + # 1. Create a table with relatively small page size. + params = 'key_format=i,value_format=S,' + self.fileConfig + self.session.create(self.uri, params) + c = self.session.open_cursor(self.uri, None) + for i in range(self.nrecords): + c[i] = self.normalValue + c.close() + + # 2. Checkpoint and get stats on the table to confirm the size. + self.session.checkpoint() + sizeWithoutOverflow = self.getSize() + self.pr('After populate ' + str(sizeWithoutOverflow // mb) + 'MB') + self.assertGreater(sizeWithoutOverflow, self.expectedTableSize * mb) + + # 3. Add overflow values. + c = self.session.open_cursor(self.uri, None) + for i in range(self.nOverflowRecords): + c[i + self.nrecords] = self.overflowValue + c.close() + + # 4. Perform checkpoint to ensure overflow values are written to the disk. + self.session.checkpoint() + sizeWithOverflow = self.getSize() + self.pr('After inserting overflow values ' + str(sizeWithoutOverflow // mb) + 'MB') + self.assertGreater(sizeWithOverflow, sizeWithoutOverflow) + + # 5. Delete middle ~90% of the normal values in the table. + if self.truncate: + c1 = self.session.open_cursor(self.uri, None) + c2 = self.session.open_cursor(self.uri, None) + c1.set_key((self.nrecords // 100) * 5) + c2.set_key((self.nrecords // 100) * 95) + self.assertEqual(self.session.truncate(None, c1, c2, None), 0) + c1.close() + c2.close() + else: + c = self.session.open_cursor(self.uri, None) + for i in range((self.nrecords // 100) * 5, (self.nrecords // 100) * 95): + c.set_key(i) + self.assertEqual(c.remove(), 0) + c.close() + + # 6. Perform checkpoint to ensure we have blocks available in the middle of the file. + self.session.checkpoint() + + # 7 & 8. Call compact. We expect that the overflow values at the end of the file are not + # rewritten and therefore the file size will mostly remain the same. Give a leeway + # of 10%. + self.session.compact(self.uri) + sizeAfterCompact = self.getSize() + self.pr('After deleting values and compactions ' + str(sizeAfterCompact // mb) + 'MB') + self.assertGreater(sizeAfterCompact, (sizeWithOverflow // 10) * 9) + + # 9. Insert some normal values and expect that file size won't increase as free extents + # in the middle of the file will be used to write new data. + + # Insert around ~50% of the normal values in the table that we deleted earlier. + c = self.session.open_cursor(self.uri, None) + for i in range((self.nrecords // 100) * 25, (self.nrecords // 100) * 75): + c.set_key(i) + c.set_value(self.normalValue) + self.assertEqual(c.update(),0) + c.close() + + # Perform compact. + self.session.compact(self.uri) + + # Test that the file size doesn't increase. + sizeAfterNewInserts = self.getSize() + self.pr('After Inserting bunch of values ' + str(sizeAfterNewInserts // mb) + 'MB') + self.assertEqual(sizeAfterCompact, sizeAfterNewInserts) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor17.py b/src/third_party/wiredtiger/test/suite/test_cursor17.py deleted file mode 100644 index 60cad85a5e2..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_cursor17.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2020 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios -from wiredtiger import stat, WT_NOTFOUND - -# test_cursor17.py -# Test the cursor traversal optimization for delete heavy workloads. This optimization enables -# cursor traversal mechanism to skip pages where all records on the page are deleted with a -# tombstone visible to the current transaction. -class test_cursor17(wttest.WiredTigerTestCase): - conn_config = 'cache_size=50MB,statistics=(all)' - session_config = 'isolation=snapshot' - - def get_stat(self, stat, uri): - stat_string = 'statistics:' - if (uri): - stat_string += uri - stat_cursor = self.session.open_cursor(stat_string) - val = stat_cursor[stat][2] - stat_cursor.close() - return val - - def test_cursor_skip_pages(self): - uri = 'table:test_cursor17' - create_params = 'key_format=i,value_format=S' - self.session.create(uri, create_params) - - value1 = 'a' * 500 - value2 = 'b' * 500 - total_keys = 40000 - - # Keep the oldest and the stable timestamp pinned. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) - cursor = self.session.open_cursor(uri) - - commit_timestamp = 3 - - # Insert predefined number of key-value pairs. - for key in range(total_keys): - self.session.begin_transaction() - cursor[key] = value1 - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) - commit_timestamp += 1 - - # Delete everything on the table except for the first and the last KV pair. - for key in range(1, total_keys - 1): - self.session.begin_transaction() - cursor.set_key(key) - self.assertEqual(cursor.remove(),0) - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) - commit_timestamp += 1 - - # Take a checkpoint to reconcile the pages. - self.session.checkpoint() - - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(commit_timestamp)) - # Position the cursor on the first record. - cursor.set_key(0) - self.assertEqual(cursor.search(), 0) - # This should move the cursor to the last record. - self.assertEqual(cursor.next(), 0) - self.assertEqual(cursor.get_key(), total_keys - 1) - - # Check if we skipped any pages while moving the cursor. - # - # We calculate the number of pages we expect to skip based on the total number of leaf pages - # reported in the WT stats. We subtract 2 from the count of leaf pages in the table and test - # that we atleast skipped 80% of the expected number of pages. - - leaf_pages_in_table = self.get_stat(stat.dsrc.btree_row_leaf, uri) - expected_pages_skipped = ((leaf_pages_in_table - 2) * 8) // 10 - skipped_pages = self.get_stat(stat.conn.cursor_next_skip_page_count, None) - self.assertGreater(skipped_pages, expected_pages_skipped) - self.session.rollback_transaction() - - # Update a key in the middle of the table. - self.session.begin_transaction() - cursor[total_keys // 2] = value2 - self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_timestamp)) - commit_timestamp += 1 - - # Make sure we can reach a the record we updated in the middle of the table. - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(commit_timestamp)) - # Position the cursor on the first record. - cursor.set_key(0) - self.assertEqual(cursor.search(), 0) - # This should move the cursor to the record we updated in the middle. - self.assertEqual(cursor.next(), 0) - self.assertEqual(cursor.get_key(), total_keys // 2) diff --git a/src/third_party/wiredtiger/test/suite/test_search_near02.py b/src/third_party/wiredtiger/test/suite/test_search_near02.py new file mode 100644 index 00000000000..a8377aac078 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_search_near02.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +import wiredtiger, wttest, unittest +from wtscenario import make_scenarios + +# test_search_near02.py +# Check the search_near API returns the correct key according to visibility rules and its +# configuration. +# When prefix search is disabled, the search_near API should always return the closest key (in +# lexicographic order) as long as one exists and is visible. +# When prefix search is enabled, in addition to the previous rules, search_near returns a key that +# matches the prefix being searched. +class test_search_near02(wttest.WiredTigerTestCase): + key_format_values = [ + ('fixed_string', dict(key_format='10s')), + ('var_string', dict(key_format='S')), + ('byte_array', dict(key_format='u')), + ] + + scenarios = make_scenarios(key_format_values) + + def check_key(self, key): + if self.key_format == 'u': + return key.encode() + elif self.key_format == '10s': + return key.ljust(10, "\x00") + else: + return key + + def test_search_near(self): + uri = 'table:test_search_near' + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + cursor = self.session.open_cursor(uri) + cursor2 = self.session.open_cursor(uri, None, "debug=(release_evict=true)") + # Basic character array. + l = "abcdefghijklmnopqrstuvwxyz" + + # Insert keys aaa -> aaz with timestamp 200. + prefix = "aa" + self.session.begin_transaction() + for k in range (0, 25): + key = prefix + l[k] + cursor[key] = key + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(200)) + + # Insert key aaz with timestamp 50. + self.session.begin_transaction() + cursor[prefix + "z"] = prefix + "z" + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) + + # Insert key aazab with timestamp 250. + self.session.begin_transaction() + cursor[prefix + "zab"] = prefix + "zab" + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(250)) + + # Evict the whole range. + for k in range (0, 26): + cursor2.set_key(prefix + l[k]) + self.assertEqual(cursor2.search(), 0) + self.assertEqual(cursor2.reset(), 0) + + cursor2.set_key(prefix + "zab") + self.assertEqual(cursor2.search(), 0) + self.assertEqual(cursor2.reset(), 0) + + # Start a transaction at timestamp 100, aaz should be the only key that is visible. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(100)) + cursor3 = self.session.open_cursor(uri) + + # Prefix search is disabled by default. + # Search near should always return the only visible key. + cursor3.set_key("aa") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + cursor3.set_key("az") + self.assertEqual(cursor3.search_near(), -1) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + cursor3.set_key("aaz") + self.assertEqual(cursor3.search_near(), 0) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + cursor3.set_key("aazab") + self.assertEqual(cursor3.search_near(), -1) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + # Enable prefix search. + cursor3.reconfigure("prefix_key=true") + + # The only visible key is aaz. As long we are looking for a key that starts with either "a", + # "aa" or "aaz", search near should return back aaz. Otherwise, search near should return + # WT_NOTFOUND. + cursor3.set_key("a") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + cursor3.set_key("aa") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + cursor3.set_key("aaz") + self.assertEqual(cursor3.search_near(), 0) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + cursor3.set_key("az") + self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) + + cursor3.set_key("b") + self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) + + cursor3.close() + self.session.commit_transaction() + + # Start a transaction at timestamp 25, no keys are visible. + # Since no keys are visible, search_near should always return WT_NOTFOUND. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(25)) + cursor3 = self.session.open_cursor(uri) + + cursor3.set_key("aaz") + self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) + + # Enable prefix search. + cursor3.reconfigure("prefix_key=true") + + cursor3.set_key("aaz") + self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) + + cursor3.close() + self.session.commit_transaction() + + # Start a transaction at timestamp 250, all keys should be visible. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(250)) + cursor3 = self.session.open_cursor(uri) + + # Search near for aa, should return the closest visible key: aaa. + cursor3.set_key("aa") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key("aaa")) + + # Search near for aaz, should return the existing visible key: aaz. + cursor3.set_key("aaz") + self.assertEqual(cursor3.search_near(), 0) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + # Search near for az, should return the closest visible key: aazab. + cursor3.set_key("az") + self.assertEqual(cursor3.search_near(), -1) + self.assertEqual(cursor3.get_key(), self.check_key("aazab")) + + # Enable prefix search. + cursor3.reconfigure("prefix_key=true") + + # Search near for a, should return the closest visible key with a matching prefix: aaa. + cursor3.set_key("a") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key("aaa")) + + # Search near for aa, should return the closest visible key with a matching prefix: aaa. + cursor3.set_key("aa") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key("aaa")) + + # Search near for aaz, should return the existing visible key with a matching prefix: aaz. + cursor3.set_key("aaz") + self.assertEqual(cursor3.search_near(), 0) + self.assertEqual(cursor3.get_key(), self.check_key("aaz")) + + # Search near for aaza, should return the closest visible key with a matching prefix: aazab. + cursor3.set_key("aaza") + self.assertEqual(cursor3.search_near(), 1) + self.assertEqual(cursor3.get_key(), self.check_key("aazab")) + + # Search near for az, no keys match the prefix, hence search_near returns WT_NOTFOUND. + cursor3.set_key("az") + self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) + cursor3.close() + self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near03.py b/src/third_party/wiredtiger/test/suite/test_search_near03.py new file mode 100644 index 00000000000..073e5c44a13 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_search_near03.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +import wiredtiger, re, wttest +from wtscenario import make_scenarios + +# test_search_near03.py +# Test prefix search near format rules, only format fixed-length strings (s), variable strings (S) +# and raw bytes array (u) is allowed. +class test_search_near03(wttest.WiredTigerTestCase): + key_format_values = [ + ('s', dict(key_format='s')), + ('5s', dict(key_format='5s')), + ('10s', dict(key_format='10s')), + ('var S', dict(key_format='S')), + ('u', dict(key_format='u')), + ('ss', dict(key_format='ss')), + ('us', dict(key_format='us')), + ('10s10s', dict(key_format='10s10s')), + ('S10s', dict(key_format='S10s')), + ('u10s', dict(key_format='u10s')), + ('var SS', dict(key_format='SS')), + ('Su', dict(key_format='Su')), + ] + + scenarios = make_scenarios(key_format_values) + + def valid_key_format(self): + if self.key_format == 'u' or self.key_format == 'S' : + return True + if re.search('^\d*s$', self.key_format): + return True + return False + + def test_prefix_reconfigure(self): + uri = 'table:test_search_near' + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + cursor = self.session.open_cursor(uri) + # Check if the format is valid for prefix configuration. + if self.valid_key_format(): + self.assertEqual(cursor.reconfigure("prefix_key=true"), 0) + else: + msg = '/Invalid argument/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.reconfigure("prefix_key=true"), msg) + cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py index f605342fa5a..5a57e95e0ae 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered03.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py @@ -49,15 +49,24 @@ class test_tiered03(wttest.WiredTigerTestCase): auth_token = "test_token" bucket = "mybucket" + absolute_bucket_dir = None # initialied in conn_config to an absolute path + cache_dir = "mybucket-cache" # a relative pathname, it will not be shared bucket_prefix = "pfx_" extension_name = "local_store" def conn_config(self): - if not os.path.exists(self.bucket): - os.mkdir(self.bucket) + # We have multiple connections that want to share a bucket directory. + # The first time this function is called, we'll establish the absolute + # path for the bucket, and always use that for the bucket name. + # The cache directory name is a relative one, so it won't be shared + # between connections. + if self.absolute_bucket_dir == None: + self.absolute_bucket_dir = os.path.join(os.getcwd(), self.bucket) + os.mkdir(self.absolute_bucket_dir) return \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ - 'bucket=%s,' % self.bucket + \ + 'bucket=%s,' % self.absolute_bucket_dir + \ + 'cache_directory=%s,' % self.cache_dir + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ 'name=%s)' % self.extension_name diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index b12bf3d12a4..614619fdef2 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -263,7 +263,7 @@ class test_tiered06(wttest.WiredTigerTestCase): os.mkdir(self.cachedir2) config1 = "cache_directory=" + self.cachedir1 config2 = "cache_directory=" + self.cachedir2 - bad_config = "cache_directory=BAD" + bad_config = "cache_directory=/BAD" # Create file system objects. First try some error cases. errmsg = '/No such file or directory/' diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp02.py b/src/third_party/wiredtiger/test/suite/test_timestamp02.py index c0f45f82dc5..4bc2c7d55ae 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp02.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp02.py @@ -130,6 +130,61 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): self.check(self.session, 'read_timestamp=' + self.timestamp_str(t + 200), dict((k, 2) for k in orig_keys[i+1:])) + # Perform validation on setting the oldest and the stable timestamps: + # - It is a success, but a no-op, to set them behind their existing values. + # - Oldest timestamp can't be more than the stable. It is reported as an error if an attempt + # is made to set that way. + # - If both the oldest and the stable are provided in the same call, the test to check if + # they are being moved backwards is done first. The value that is being set backwards is + # silently dropped, as if not provided at all. This is followed by the test on the oldest + # not being being set ahead of the stable. + + # Confirm that the oldest and the stable timestamps are as expected + self.assertTimestampsEqual(self.conn.query_timestamp("get=oldest"), self.timestamp_str(200)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=stable"), self.timestamp_str(300)) + + # Any attempt to set the oldest or stable to a value older than the current is silently + # ignored. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=oldest"), self.timestamp_str(200)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=stable"), self.timestamp_str(300)) + + # An error to set oldest ahead of stable. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(301)), + '/oldest timestamp \(0, 301\) must not be later than stable timestamp \(0, 300\)/') + self.assertTimestampsEqual(self.conn.query_timestamp("get=oldest"), self.timestamp_str(200)) + + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(201)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=oldest"), self.timestamp_str(201)) + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(301)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=stable"), self.timestamp_str(301)) + + # If both the oldest and the stable are provided in the same call, the behavior should be + # consistent with providing them individually. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=oldest"), self.timestamp_str(201)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=stable"), self.timestamp_str(301)) + + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(202) + + ',stable_timestamp=' + self.timestamp_str(1)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=oldest"), self.timestamp_str(202)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=stable"), self.timestamp_str(301)) + + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(302) + + ',stable_timestamp=' + self.timestamp_str(1)), + '/oldest timestamp \(0, 302\) must not be later than stable timestamp \(0, 301\)/') + self.assertTimestampsEqual(self.conn.query_timestamp("get=oldest"), self.timestamp_str(202)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=stable"), self.timestamp_str(301)) + + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(302) + + ',stable_timestamp=' + self.timestamp_str(302)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=oldest"), self.timestamp_str(302)) + self.assertTimestampsEqual(self.conn.query_timestamp("get=stable"), self.timestamp_str(302)) + def test_read_your_writes(self): self.session.create(self.uri, 'key_format=i,value_format=i' + self.extra_config) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py index e271c8f1145..17868e1b821 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp19.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -98,12 +98,6 @@ class test_timestamp19(wttest.WiredTigerTestCase): self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'), self.timestamp_str(40)) - # Trying to set an earlier stable timestamp is an error. - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)), - '/oldest timestamp \(0, 40\) must not be later than stable timestamp \(0, 10\)/') - self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'), self.timestamp_str(40)) - # Move the oldest and stable timestamps to 70. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(70) + ', stable_timestamp=' + self.timestamp_str(70)) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index b26b844dd32..55d6329af2f 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -328,6 +328,14 @@ class test_timestamp22(wttest.WiredTigerTestCase): return expected expected = self.SUCCESS + + # It is a no-op to provide oldest or stable behind the global values. If provided ahead, we + # will treat the values as if not provided at all. + if oldest <= self.oldest_ts: + oldest = -1 + if stable <= self.stable_ts: + stable = -1 + if oldest >= 0 and stable < 0: expected = expected_newer(expected, self.stable_ts, oldest, self.oldest_ts) expected = expected_newer(expected, stable, oldest, self.oldest_ts) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp24.py b/src/third_party/wiredtiger/test/suite/test_timestamp24.py new file mode 100644 index 00000000000..ec5814e301a --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp24.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_timestamp24.py +# +# Make sure certain conflicting operations are rejected. + +class test_timestamp24(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + + def evict(self, uri, session, key, value): + evict_cursor = session.open_cursor(uri, None, "debug=(release_evict)") + session.begin_transaction() + v = evict_cursor[key] + self.assertEqual(v, value) + self.assertEqual(evict_cursor.reset(), 0) + session.rollback_transaction() + + def test_timestamp(self): + + table_uri = 'table:timestamp24' + ds = SimpleDataSet( + self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + key = 5 + value_a = 'a' * 500 + value_b = 'b' * 500 + value_c = 'c' * 500 + value_d = 'd' * 500 + + # Pin oldest and stable to timestamp 1. + #self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + # ',stable_timestamp=' + self.timestamp_str(1)) + + # Create two sessions so we can have two transactions. + session1 = self.session + session2 = self.conn.open_session() + + # In session 1, write value_a at time 20. + # Commit that and then start a new transaction. Read the value back. + # Then reset the cursor so the page isn't pinned, but leave the transaction open. + cursor1 = session1.open_cursor(ds.uri) + session1.begin_transaction() + cursor1[key] = value_a + session1.commit_transaction('commit_timestamp=20') + session1.begin_transaction('read_timestamp=25') + tmp = cursor1[key] + self.assertEqual(tmp, value_a) + cursor1.reset() + # leave session1's transaction open + + # In session 2, write value_b at time 50. Commit that. + cursor2 = session2.open_cursor(ds.uri) + session2.begin_transaction() + cursor2[key] = value_b + session2.commit_transaction('commit_timestamp=50') + cursor2.reset() + + # Evict the page to force reconciliation. value_b goes to disk; value_a to history. + # Use session2 so we can keep session1's transaction open. + self.evict(ds.uri, session2, key, value_b) + + # In session 2, write value_c, but abort it. + session2.begin_transaction() + cursor2[key] = value_c + session2.rollback_transaction() + + # Now in session 1 try to write value_d. This should produce WT_ROLLBACK, but with + # a bug seen and fixed in August 2021, succeeds improperly instead, resulting in + # data corruption. The behavior is more exciting when the update is a modify (the + # modify gets applied to value_b instead of value_a, producing a more detectable + # corruption) but this is not necessary to check the wrong behavior. + + try: + cursor1[key] = value_d + self.fail("Conflicting update did not fail") + broken = True + except wiredtiger.WiredTigerError as e: + self.assertTrue(wiredtiger.wiredtiger_strerror(wiredtiger.WT_ROLLBACK) in str(e)) + broken = False + + # Put this outside the try block in case it throws its own exceptions + if broken: + session1.commit_transaction('commit_timestamp=30') + else: + session1.rollback_transaction() + + # Read the data back + session2.begin_transaction('read_timestamp=60') + tmp = cursor2[key] + + # It should be value_b. But if we broke, it'll be value_d. + self.assertEqual(tmp, value_d if broken else value_b) + + cursor2.close() + cursor1.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_verify2.py b/src/third_party/wiredtiger/test/suite/test_verify2.py new file mode 100755 index 00000000000..65163191b8f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_verify2.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +class test_verify2(wttest.WiredTigerTestCase): + tablename = 'test_verify' + params = 'key_format=S,value_format=S' + uri = 'table:' + tablename + + # Create an empty table and insert content. + # The first call to verify is expected to return to EBUSY due to the dirty content. Call + # checkpoint to make the table clean, the next verify call should succeed. + def test_verify_ckpt(self): + self.assertEqual(self.session.create(self.uri, self.params), 0) + self.assertEqual(self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)), 0) + + # Insert data. + cursor = self.session.open_cursor(self.uri) + cursor["0"] = "000" + cursor.close() + + # Calling verify without checkpointing before will return EBUSY because of the dirty data. + self.assertTrue(self.raisesBusy(lambda: self.session.verify(self.uri, None)), + "was expecting API call to fail with EBUSY") + + # Checkpointing will get rid of the dirty data. + self.assertEqual(self.session.checkpoint(), 0) + + # Verify. + self.assertEqual(self.session.verify(self.uri, None), 0) + + # Create an empty table and search a key. This used to mark the associated btree as dirty. In + # fact, because the tree is empty, its only reference to a leaf page is marked as deleted and we + # instantiate the deleted page in this case. Before WT-8126, this would mark the btree as + # modified. + def test_verify_search(self): + self.assertEqual(self.session.create(self.uri, self.params), 0) + self.assertEqual(self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)), 0) + + # Search for data. + cursor = self.session.open_cursor(self.uri) + cursor.set_key("1") + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + cursor.close() + + # We don't need to call checkpoint before calling verify as the btree is not marked as + # modified. + self.assertEqual(self.session.verify(self.uri, None), 0) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 16f79426533..c39b189a634 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -250,7 +250,7 @@ testutil_copy_data(const char *dir) char buf[512]; testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && cp -p * ../%s.SAVE", dir, dir, dir)); + "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && cp -rp * ../%s.SAVE", dir, dir, dir)); if ((status = system(buf)) < 0) testutil_die(status, "system: %s", buf); } diff --git a/src/third_party/wiredtiger/tools/run_parallel.sh b/src/third_party/wiredtiger/tools/run_parallel.sh new file mode 100755 index 00000000000..526dc5e65f0 --- /dev/null +++ b/src/third_party/wiredtiger/tools/run_parallel.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +usage () { + cat << EOF +Usage: run_parallel.sh {command} {num_iter} {num_parallel} +Where: + {command} is a string containing the command, including parameters, to run + {num_iter} is an positive integer indicating how many iterations to execute + [num_parallel] is an (optional) positive integer indicating how many parallel commands should be executed + in each iteration. If not provided, the default is half the number of available CPU cores. +EOF +} + +if [ "$#" -lt 2 ]; then + echo "Illegal number of parameters." + usage + exit 1 +fi + +# Determine the number of CPU cores. This code is Linux specific at the moment. +NCORES=$(grep -c ^processor /proc/cpuinfo) + +command=$1 +num_iter=$2 + +if [ "$#" -eq 3 ]; then + num_parallel=$3 +else + # Use half the number of processor cores + num_parallel="$(($NCORES / 2))" +fi + +echo "run_parallel:" +echo " number of cores: $NCORES" +echo " command: $command" +echo " num_parallel: $num_parallel" +echo " num_iter: $num_iter" + +outf=./outfile.txt + +for i in $(seq $num_iter); do + echo "Starting iteration $i" >> $outf + echo "Starting iteration $i" + + # start the commands in parallel + for((t=1; t<=num_parallel; t++)); do + echo "Starting parallel command $t (of $num_parallel) in iteration $i (of $num_iter)" >> nohup.out.$t + eval nohup $command >> nohup.out.$t 2>&1 & + done + + # Wait for the commands to all complete + for((t=1; t<=num_parallel; t++)); do + wait || exit $? + done +done |
