diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-08-20 12:46:38 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-08-20 03:06:59 +0000 |
| commit | d77ff77250a931dedd382956cf54fe40d815dfda (patch) | |
| tree | c3d0ccab30fecd901927bcd6706d6379c2caaf54 | |
| parent | 0c6300fedd5fa9c074fa4c97f5e4193cd7645593 (diff) | |
| download | mongo-d77ff77250a931dedd382956cf54fe40d815dfda.tar.gz | |
Import wiredtiger: 27d10c3362673e12fb53b9c4be22cbb9fd46c72c from branch mongodb-4.4
ref: 03eb45ea7f..27d10c3362
for: 4.4.9
WT-6915 Write "log files" subpage for Architecture Guide
WT-7363 Add support for dumping history store output in hex format
WT-7663 Change local store extension to allow only readonly FS operations
WT-7718 Rename 'build_cmake'
WT-7866 Update cache_hs_insert limits in cppsuite-hs-cleanup-stress
WT-7880 Fix history store record issue when the update following the prepared update is in history store
WT-7891 Remove doc typos
WT-7907 Add dependencies to swig module definition in CMake build
WT-7918 Support setting a prepare timestamp at current read timestamp
WT-7928 VLCS checkpoint and additional test suite improvements
WT-7934 Upload perf results from many-collection test to Altas
WT-7936 Update the format.sh script to run recovery tests
WT-7940 Update mongod path for many-coll test
WT-7943 Do not assert timestamps when rolling back a prepared transactions
WT-7945 Move rollback handling to the operation layer in the cppsuite.
WT-7947 Allow CMake to take in a specific Python version
WT-7952 Minor docs build fixes
WT-7953 Teach s_string to not look inside getopt option lists.
WT-7955 Copy format.sh and CONFIG.stress to the test/format build directory with CMake
WT-7956 RTS to skip deleted or stable RLE cells
WT-7961 Sometimes lag oldest timestamp in timestamp_abort.
116 files changed, 1331 insertions, 805 deletions
diff --git a/src/third_party/wiredtiger/build_cmake/README.md b/src/third_party/wiredtiger/cmake/README.md index caea84aec53..75fa960da6a 100644 --- a/src/third_party/wiredtiger/build_cmake/README.md +++ b/src/third_party/wiredtiger/cmake/README.md @@ -113,24 +113,24 @@ $ cd build $ ccmake . ``` -*The configuration options can also be viewed in `build_cmake/configs/base.cmake`*. +*The configuration options can also be viewed in `cmake/configs/base.cmake`*. ###### Switching between GCC and Clang (POSIX only) -By default CMake will use your default system compiler (`cc`). If you want to use a specific toolchain you can pass a toolchain file! We have provided a toolchain file for both GCC (`build_cmake/toolchains/gcc.cmake`) and Clang (`build_cmake/toolchains/clang.cmake`). To use either toolchain you can pass the `-DCMAKE_TOOLCHAIN_FILE=` to the CMake configuration step. For example: +By default CMake will use your default system compiler (`cc`). If you want to use a specific toolchain you can pass a toolchain file! We have provided a toolchain file for both GCC (`cmake/toolchains/gcc.cmake`) and Clang (`cmake/toolchains/clang.cmake`). To use either toolchain you can pass the `-DCMAKE_TOOLCHAIN_FILE=` to the CMake configuration step. For example: *Using the GCC Toolchain* ```bash $ cd build -$ cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/gcc.cmake -G Ninja ../. +$ cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/gcc.cmake -G Ninja ../. ``` *Using the Clang Toolchain* ```bash $ cd build -$ cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/clang.cmake -G Ninja ../. +$ cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -G Ninja ../. ``` ### Running WiredTiger C Tests diff --git a/src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake index 94447ac9ca8..94447ac9ca8 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake +++ b/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake diff --git a/src/third_party/wiredtiger/build_cmake/configs/auto.cmake b/src/third_party/wiredtiger/cmake/configs/auto.cmake index b9026d90d70..3691da331a8 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/auto.cmake +++ b/src/third_party/wiredtiger/cmake/configs/auto.cmake @@ -6,7 +6,7 @@ # See the file LICENSE for redistribution information # -include(build_cmake/helpers.cmake) +include(cmake/helpers.cmake) ### Auto configure options and checks that we can infer from our toolchain environment. diff --git a/src/third_party/wiredtiger/build_cmake/configs/base.cmake b/src/third_party/wiredtiger/cmake/configs/base.cmake index 2b44a7c85cb..098859f2734 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/base.cmake +++ b/src/third_party/wiredtiger/cmake/configs/base.cmake @@ -6,7 +6,7 @@ # See the file LICENSE for redistribution information # -include(build_cmake/helpers.cmake) +include(cmake/helpers.cmake) # WiredTiger-related configuration options. @@ -72,6 +72,16 @@ config_bool( DEFAULT OFF ) +config_string( + PYTHON3_REQUIRED_VERSION + "Exact Python version to use when building the Python API. \ + By default, when this configuration is unset, CMake will preference the \ + highest python version found to be installed in the users system path. \ + Expected format of version string: major[.minor[.patch]]" + DEFAULT "" + DEPENDS "ENABLE_PYTHON" +) + config_bool( WT_STANDALONE_BUILD "Support standalone build" diff --git a/src/third_party/wiredtiger/build_cmake/configs/compile_test/pthread_cond_monotonic_test.c b/src/third_party/wiredtiger/cmake/configs/compile_test/pthread_cond_monotonic_test.c index 64645ec889f..64645ec889f 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/compile_test/pthread_cond_monotonic_test.c +++ b/src/third_party/wiredtiger/cmake/configs/compile_test/pthread_cond_monotonic_test.c diff --git a/src/third_party/wiredtiger/build_cmake/configs/modes.cmake b/src/third_party/wiredtiger/cmake/configs/modes.cmake index 74bf5cf8188..74bf5cf8188 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/modes.cmake +++ b/src/third_party/wiredtiger/cmake/configs/modes.cmake diff --git a/src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/ppc64le/linux/config.cmake index 4d1821c9e18..4d1821c9e18 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake +++ b/src/third_party/wiredtiger/cmake/configs/ppc64le/linux/config.cmake diff --git a/src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/s390x/linux/config.cmake index 639da9f297e..639da9f297e 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake +++ b/src/third_party/wiredtiger/cmake/configs/s390x/linux/config.cmake diff --git a/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in b/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in index 5cdf5d357b0..5cdf5d357b0 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in +++ b/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/darwin/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake index 2d3f7ead67a..2d3f7ead67a 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/darwin/config.cmake +++ b/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake index 4cd2d6882e4..4cd2d6882e4 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake +++ b/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/netbsd/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/netbsd/config.cmake index f307d3b0110..f307d3b0110 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/netbsd/config.cmake +++ b/src/third_party/wiredtiger/cmake/configs/x86/netbsd/config.cmake diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/windows/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake index 842708a1f9d..842708a1f9d 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/windows/config.cmake +++ b/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake diff --git a/src/third_party/wiredtiger/build_cmake/helpers.cmake b/src/third_party/wiredtiger/cmake/helpers.cmake index 4dc449c0748..f3019627b8e 100644 --- a/src/third_party/wiredtiger/build_cmake/helpers.cmake +++ b/src/third_party/wiredtiger/cmake/helpers.cmake @@ -61,10 +61,6 @@ function(config_string config_name description) if (NOT "${CONFIG_STR_UNPARSED_ARGUMENTS}" STREQUAL "") message(FATAL_ERROR "Unknown arguments to config_str: ${CONFIG_STR_UNPARSED_ARGUMENTS}") endif() - # We require a default value (not optional). - if ("${CONFIG_STR_DEFAULT}" STREQUAL "") - message(FATAL_ERROR "No default value passed") - endif() # Check that the configs dependencies are enabled before setting it to a visible enabled state. eval_dependency("${CONFIG_STR_DEPENDS}" enabled) diff --git a/src/third_party/wiredtiger/build_cmake/install/install.cmake b/src/third_party/wiredtiger/cmake/install/install.cmake index ec9b30234d6..ec9b30234d6 100644 --- a/src/third_party/wiredtiger/build_cmake/install/install.cmake +++ b/src/third_party/wiredtiger/cmake/install/install.cmake diff --git a/src/third_party/wiredtiger/build_cmake/install/wiredtiger.pc.in b/src/third_party/wiredtiger/cmake/install/wiredtiger.pc.in index 9ed407b7fe4..9ed407b7fe4 100644 --- a/src/third_party/wiredtiger/build_cmake/install/wiredtiger.pc.in +++ b/src/third_party/wiredtiger/cmake/install/wiredtiger.pc.in diff --git a/src/third_party/wiredtiger/build_cmake/strict/cl_strict.cmake b/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake index 625bf9e88d5..625bf9e88d5 100644 --- a/src/third_party/wiredtiger/build_cmake/strict/cl_strict.cmake +++ b/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake diff --git a/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake b/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake index f87c988fe02..f87c988fe02 100644 --- a/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake +++ b/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake diff --git a/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake b/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake index 420a4457676..420a4457676 100644 --- a/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake +++ b/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_clang.cmake index 98cc2a8892b..98cc2a8892b 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_clang.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_gcc.cmake index b82bbfe9b94..b82bbfe9b94 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_gcc.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/cl.cmake b/src/third_party/wiredtiger/cmake/toolchains/cl.cmake index d6467238ce3..d6467238ce3 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/cl.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/cl.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/clang.cmake index 5b789f1134f..5b789f1134f 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/clang.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/gcc.cmake index 0fa64ed0037..0fa64ed0037 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/gcc.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_clang.cmake index 2353ba1b7f7..2353ba1b7f7 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_clang.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_clang.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_gcc.cmake index 136d4a33f4f..136d4a33f4f 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_gcc.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_clang.cmake index f03d4fdbd97..f03d4fdbd97 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_clang.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_gcc.cmake index 47e706c0d39..47e706c0d39 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_gcc.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_clang.cmake index f643b9c8df8..f643b9c8df8 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_clang.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_gcc.cmake index abfff342dfb..abfff342dfb 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_gcc.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_clang.cmake index e7130be256f..e7130be256f 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_clang.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_gcc.cmake index 7a463e46d96..7a463e46d96 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_gcc.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_clang.cmake index 126a3a1bbbe..126a3a1bbbe 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_clang.cmake diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_gcc.cmake index 2b36fb667d5..2b36fb667d5 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake +++ b/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_gcc.cmake diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 9663188789b..4b4c98f6018 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1432,10 +1432,11 @@ methods = { selects a simple hexadecimal format, "json" selects a JSON format with each record formatted as fields named by column names if available, "pretty" selects a human-readable format (making it - incompatible with the "load") and "print" selects a format where only - non-printing characters are hexadecimal encoded. These formats are - compatible with the @ref util_dump and @ref util_load commands''', - choices=['hex', 'json', 'pretty', 'print']), + incompatible with the "load"), "pretty_hex" is similar to "pretty" (also incompatible with + "load") except raw byte data elements will be printed like "hex" format, and + "print" selects a format where only non-printing characters are hexadecimal encoded. These + formats are compatible with the @ref util_dump and @ref util_load commands''', + choices=['hex', 'json', 'pretty', 'pretty_hex', 'print']), Config('incremental', '', r''' configure the cursor for block incremental backup usage. These formats are only compatible with the backup data source; see @ref backup''', @@ -1659,10 +1660,11 @@ methods = { applicable only for prepared transactions. Indicates if the prepare timestamp and the commit timestamp of this transaction can be rounded up. If the prepare timestamp is less than the oldest - timestamp, the prepare timestamp will be rounded to the oldest + timestamp, the prepare timestamp will be rounded to the oldest timestamp. If the commit timestamp is less than the prepare timestamp, the commit timestamp will be rounded up to the prepare - timestamp''', type='boolean'), + timestamp. Allows setting the prepared timestamp smaller than or equal + to the latest active read timestamp''', type='boolean'), Config('read', 'false', r''' if the read timestamp is less than the oldest timestamp, the read timestamp will be rounded up to the oldest timestamp''', diff --git a/src/third_party/wiredtiger/dist/s_copyright.list b/src/third_party/wiredtiger/dist/s_copyright.list index f1c879a555b..8f4534c2670 100644 --- a/src/third_party/wiredtiger/dist/s_copyright.list +++ b/src/third_party/wiredtiger/dist/s_copyright.list @@ -1,6 +1,6 @@ skip bench/workgen/workgen/workgen.py skip bench/workgen/workgen_wrap.cxx -skip build_cmake/install/wiredtiger.pc.in +skip cmake/install/wiredtiger.pc.in skip build_win/wiredtiger_config.h skip dist/api_config.py skip dist/api_config_gen.py diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 83b31aa1a09..8b5c02aa8a8 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -57,7 +57,7 @@ wtperf_config() glossarychk() { (cd ../src/docs && grep '^<tr><td>' arch-glossary.dox | cut -d'>' -f3 | sed -e 's/<.*//') > $t - sort --ignore-case $t > $t2 + sort -f $t > $t2 if ! diff $t $t2; then echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" echo "arch-glossary.dox entries out of order" diff --git a/src/third_party/wiredtiger/dist/s_export b/src/third_party/wiredtiger/dist/s_export index 0df4f16b3ef..8615f1415b2 100755 --- a/src/third_party/wiredtiger/dist/s_export +++ b/src/third_party/wiredtiger/dist/s_export @@ -51,7 +51,7 @@ check() # ../build_posix/.libs/libwiredtiger.$ext (autotools build) # ../.libs/libwiredtiger.$ext (autotools build at top level) # ../build/libwiredtiger.$ext (cmake build) -# ../build_cmake/libwiredtiger.$ext (cmake build) +# ../cmake_build/libwiredtiger.$ext (cmake build) found=0 for f in $(find .. -name "libwiredtiger.*" -print); do diff --git a/src/third_party/wiredtiger/dist/s_string b/src/third_party/wiredtiger/dist/s_string index 4aab8a16f38..c41f0d82381 100755 --- a/src/third_party/wiredtiger/dist/s_string +++ b/src/third_party/wiredtiger/dist/s_string @@ -32,7 +32,8 @@ replace() { check() { # Strip out git hashes, which are seven character hex strings. # Strip out double quote char literals ('"'), they confuse aspell. - sed -e 's/ [0-9a-f]\{7\} / /g' -e "s/'\"'//g" ../$2 | + # Strip out calls to __wt_getopt so the option lists don't have to be spelling words. + sed -e 's/ [0-9a-f]\{7\} / /g' -e "s/'\"'//g" -e 's/__wt_getopt([^()]*)//' ../$2 | aspell --lang=en_US $1 list | sort -u | comm -23 /dev/stdin s_string.ok > $t diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index d0b2ab55adf..1b4cf50716e 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -113,7 +113,6 @@ Decrement Decrypt DeleteFileW Destructor -Dh EACCES EAGAIN EB @@ -156,7 +155,6 @@ FindClose FindFirstFile FindNextFileW Fixup -Fk FlushFileBuffers Fprintf FreeBSD @@ -244,7 +242,6 @@ LevelDB Levyx LibFuzzer LmRrSVv -LmsT LoadLoad LockFile Lookaside @@ -498,7 +495,6 @@ abcdef abcdefghijklmnopqrstuvwxyz addl addr -af agc alfred alloc @@ -587,7 +583,6 @@ ccc ccr cd ce -ceh celsius centric cfg @@ -962,7 +957,6 @@ iter iteratively iters jjj -jnr jprx json kb @@ -1036,7 +1030,6 @@ lwsync lx lz lzo -mT madvise majmin majorp @@ -1159,8 +1152,6 @@ osfhandle other's ovfl ownp -pR -pS packv pagedump pagesize @@ -1218,7 +1209,6 @@ pv pvA pwrite py -qRrT qdown qqq qrrSS @@ -1279,7 +1269,6 @@ runtime rwlock sH sHQ -sT sanitizer sanitizers scalability @@ -1472,7 +1461,6 @@ util utils uu vN -vW va valgrind valuep @@ -1493,10 +1481,7 @@ vsize vsnprintf vtype vunpack -vw vxr -vxz -vz waitpid waker wakeup diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 32d3d237858..f35e56252a1 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -876,6 +876,7 @@ conn_dsrc_stats = [ ########################################## TxnStat('txn_checkpoint_obsolete_applied', 'transaction checkpoints due to obsolete pages'), TxnStat('txn_read_race_prepare_update', 'race to read prepared update retry'), + TxnStat('txn_rts_delete_rle_skipped', 'rollback to stable skipping delete rle'), TxnStat('txn_rts_hs_removed', 'rollback to stable updates removed from history store'), TxnStat('txn_rts_hs_restore_updates', 'rollback to stable restored updates from history store'), TxnStat('txn_rts_hs_restore_tombstones', 'rollback to stable restored tombstones from history store'), @@ -883,6 +884,7 @@ conn_dsrc_stats = [ TxnStat('txn_rts_inconsistent_ckpt', 'rollback to stable inconsistent checkpoint'), TxnStat('txn_rts_keys_removed', 'rollback to stable keys removed'), TxnStat('txn_rts_keys_restored', 'rollback to stable keys restored'), + TxnStat('txn_rts_stable_rle_skipped', 'rollback to stable skipping stable rle'), TxnStat('txn_rts_sweep_hs_keys', 'rollback to stable sweeping history store keys'), TxnStat('txn_update_conflict', 'update conflicts'), ] diff --git a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c index eeca2a83b99..11384645222 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c +++ b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c @@ -83,9 +83,10 @@ typedef struct { /* * Statistics are collected but not yet exposed. */ - uint64_t fh_ops; /* Non-read/write operations in file handles */ - uint64_t object_flushes; /* (What would be) writes to the cloud */ - uint64_t op_count; /* Number of operations done on local */ + uint64_t fh_ops; /* Non-read/write operations in file handles */ + uint64_t object_writes; /* (What would be) writes to the cloud */ + uint64_t object_reads; /* (What would be) reads to the cloud */ + uint64_t op_count; /* Number of operations done on local */ uint64_t read_ops; uint64_t write_ops; @@ -129,7 +130,8 @@ static int local_file_copy( LOCAL_STORAGE *, WT_SESSION *, const char *, const char *, WT_FS_OPEN_FILE_TYPE); static int local_get_directory(const char *, ssize_t len, char **); static int local_path(WT_FILE_SYSTEM *, const char *, const char *, char **); -static int local_writeable(LOCAL_STORAGE *, const char *name, bool *writeable); +static int local_stat( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, bool, struct stat *); /* * Forward function declarations for storage source API implementation @@ -243,10 +245,12 @@ local_delay(LOCAL_STORAGE *local) int ret; ret = 0; - if (local->force_delay != 0 && local->object_flushes % local->force_delay == 0) { + if (local->force_delay != 0 && + (local->object_reads + local->object_writes) % local->force_delay == 0) { WT_VERBOSE_LS(local, - "Artificial delay %" PRIu32 " milliseconds after %" PRIu64 " object flushes\n", - local->delay_ms, local->object_flushes); + "Artificial delay %" PRIu32 " milliseconds after %" PRIu64 " object reads, %" PRIu64 + " object writes\n", + local->delay_ms, local->object_reads, local->object_writes); /* * tv_usec has type suseconds_t, which is signed (hence the s), but ->delay_ms is unsigned. * In both gcc8 and gcc10 with -Wsign-conversion enabled (as we do) this causes a spurious @@ -257,9 +261,11 @@ local_delay(LOCAL_STORAGE *local) tv.tv_usec = (suseconds_t)(local->delay_ms % 1000) * 1000; (void)select(0, NULL, NULL, NULL, &tv); } - if (local->force_error != 0 && local->object_flushes % local->force_error == 0) { - WT_VERBOSE_LS(local, "Artificial error returned after %" PRIu64 " object flushes\n", - local->object_flushes); + if (local->force_error != 0 && + (local->object_reads + local->object_writes) % local->force_error == 0) { + WT_VERBOSE_LS(local, + "Artificial error returned after %" PRIu64 " object reads, %" PRIu64 " object writes\n", + local->object_reads, local->object_writes); ret = ENETUNREACH; } @@ -317,28 +323,6 @@ local_get_directory(const char *s, ssize_t len, char **copy) } /* - * local_writeable -- - * Check if a file can be written, or equivalently, check to see that it has not been flushed. - * This will be true if it is in the regular file system (not one managed by local_store). - */ -static int -local_writeable(LOCAL_STORAGE *local, const char *name, bool *writeablep) -{ - struct stat sb; - int ret; - - ret = 0; - *writeablep = false; - - if (stat(name, &sb) == 0) - *writeablep = true; - else if (errno != ENOENT) - ret = local_err(local, NULL, errno, "%s: stat", name); - - return (ret); -} - -/* * local_bucket_path -- * Construct the bucket pathname from the file system and local name. */ @@ -388,6 +372,49 @@ local_path(WT_FILE_SYSTEM *file_system, const char *dir, const char *name, char } /* + * local_stat -- + * Perform the stat system call for a name in the file system. + */ +static int +local_stat(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, const char *caller, + bool must_exist, struct stat *statp) +{ + int ret; + char *path; + + path = NULL; + + /* + * We check to see if the file exists in the cache first, and if not the bucket directory. This + * maps what a real cloud implementation would do. This will allow us to instrument this code to + * try out and measure caching implementations. + */ + if ((ret = local_cache_path(file_system, name, &path)) != 0) + goto err; + + ret = stat(path, statp); + if (ret != 0 && errno == ENOENT) { + /* It's not in the cache, try the bucket directory. */ + free(path); + if ((ret = local_bucket_path(file_system, name, &path)) != 0) + goto err; + ret = stat(path, statp); + } + if (ret != 0) { + /* + * If the file must exist, report the error no matter what. + */ + if (must_exist || errno != ENOENT) + ret = local_err(WT_FS2LOCAL(file_system), session, errno, "%s: %s stat", path, caller); + else + ret = errno; + } +err: + free(path); + return (ret); +} + +/* * local_add_reference -- * Add a reference to the storage source so we can reference count to know when to really * terminate. @@ -517,34 +544,16 @@ local_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, struct stat sb; LOCAL_STORAGE *local; int ret; - char *path; local = WT_FS2LOCAL(file_system); - path = NULL; - - /* If the file exists directly in the file system, it's not yet flushed, and we're done. */ - ret = stat(name, &sb); - if (ret == 0) { - *existp = true; - return (0); - } else if (errno != ENOENT) - ret = local_err(local, session, errno, "%s: ss_exist stat", path); - local->op_count++; - if ((ret = local_cache_path(file_system, name, &path)) != 0) - goto err; + *existp = false; - ret = stat(path, &sb); - if (ret == 0) + if ((ret = local_stat(file_system, session, name, "ss_exist", false, &sb)) == 0) *existp = true; - else if (errno == ENOENT) { + else if (ret == ENOENT) ret = 0; - *existp = false; - } else - ret = local_err(local, session, errno, "%s: ss_exist stat", path); -err: - free(path); return (ret); } @@ -636,7 +645,7 @@ local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, WT_FILE_SYST if ((ret = local_file_copy(local, session, source, dest_path, WT_FS_OPEN_FILE_TYPE_DATA)) != 0) goto err; - local->object_flushes++; + local->object_writes++; err: free(dest_path); @@ -781,14 +790,14 @@ local_directory_list_internal(WT_FILE_SYSTEM *file_system, WT_SESSION *session, *countp = 0; /* - * We list items in the cache directory (these have 'finished' flushing). + * The items in the bucket directory represent the definitive list. */ - if ((dirp = opendir(local_fs->cache_dir)) == NULL) { + if ((dirp = opendir(local_fs->bucket_dir)) == NULL) { ret = errno; if (ret == 0) ret = EINVAL; return ( - local_err(local, session, ret, "%s: ss_directory_list: opendir", local_fs->cache_dir)); + local_err(local, session, ret, "%s: ss_directory_list: opendir", local_fs->bucket_dir)); } for (count = 0; (dp = readdir(dirp)) != NULL && (limit == 0 || count < limit);) { @@ -868,9 +877,7 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, WT_FILE_SYSTEM *wt_fs; struct stat sb; int ret; - char *alloced_path; - const char *path; - bool create, exists; + char *bucket_path, *cache_path; (void)flags; /* Unused */ @@ -880,7 +887,11 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, local_fs = (LOCAL_FILE_SYSTEM *)file_system; local = local_fs->local_storage; wt_fs = local_fs->wt_fs; - alloced_path = NULL; + bucket_path = cache_path = NULL; + + if ((flags & WT_FS_OPEN_READONLY) == 0 || (flags & WT_FS_OPEN_CREATE) != 0) + return ( + local_err(local, session, EINVAL, "ss_open_object: readonly access required: %s", name)); /* * We expect that the local file system will be used narrowly, like when creating or opening a @@ -902,41 +913,32 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, ret = ENOMEM; goto err; } - create = ((flags & WT_FS_OPEN_CREATE) != 0); - if (!create) { - ret = stat(name, &sb); - if (ret != 0 && errno != ENOENT) { - ret = local_err(local, session, errno, "%s: local_open stat", name); + if ((ret = local_cache_path(file_system, name, &cache_path)) != 0) + goto err; + ret = stat(cache_path, &sb); + if (ret != 0) { + if (errno != ENOENT) { + ret = local_err(local, session, errno, "%s: local_open stat", cache_path); goto err; } - exists = (ret == 0); - } else - exists = false; - if (create || exists) - /* The file has not been flushed, use the file directly in the file system. */ - path = name; - else { - if ((ret = local_cache_path(file_system, name, &alloced_path)) != 0) + + /* + * The file doesn't exist locally, make a copy of it from the cloud. + */ + if ((ret = local_bucket_path(file_system, name, &bucket_path)) != 0) goto err; - path = alloced_path; - ret = stat(path, &sb); - if (ret != 0 && errno != ENOENT) { - ret = local_err(local, session, errno, "%s: local_open stat", path); + + if ((ret = local_delay(local)) != 0) + goto err; + + if ((ret = local_file_copy( + local, session, bucket_path, cache_path, WT_FS_OPEN_FILE_TYPE_DATA)) != 0) goto err; - } - exists = (ret == 0); - } - /* - * TODO: tiered: If the file doesn't exist locally, make a copy of it from the cloud here. - * - */ -#if 0 - if ((flags & WT_FS_OPEN_READONLY) != 0 && !exists) { - } -#endif - if ((ret = wt_fs->fs_open_file(wt_fs, session, path, file_type, flags, &wt_fh)) != 0) { - ret = local_err(local, session, ret, "ss_open_object: open: %s", path); + local->object_reads++; + } + if ((ret = wt_fs->fs_open_file(wt_fs, session, cache_path, file_type, flags, &wt_fh)) != 0) { + ret = local_err(local, session, ret, "ss_open_object: open: %s", name); goto err; } local_fh->fh = wt_fh; @@ -985,7 +987,8 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, WT_SHOW_STRING(local_fh->fh->name)); err: - free(alloced_path); + free(bucket_path); + free(cache_path); if (ret != 0) { if (local_fh != NULL) local_file_close_internal(local, session, local_fh); @@ -995,75 +998,30 @@ err: /* * local_rename -- - * POSIX rename, for files not yet flushed to the cloud. If a file has been flushed, we don't - * support this operation. That is because cloud implementations may not support it, and more - * importantly, we consider anything in the cloud to be readonly as far as the custom file - * system is concerned. + * POSIX rename, not supported for cloud objects. */ static int local_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *from, const char *to, uint32_t flags) { - LOCAL_FILE_SYSTEM *local_fs; - LOCAL_STORAGE *local; - WT_FILE_SYSTEM *wt_fs; - int ret; - bool writeable; - - local = WT_FS2LOCAL(file_system); - local_fs = (LOCAL_FILE_SYSTEM *)file_system; - wt_fs = local_fs->wt_fs; - - local->op_count++; - if ((ret = local_writeable(local, from, &writeable)) != 0) - goto err; - if (!writeable) { - ret = local_err(local, session, ENOTSUP, "%s: rename of flushed file not allowed", from); - goto err; - } - - if ((ret = wt_fs->fs_rename(wt_fs, session, from, to, flags)) != 0) { - ret = local_err(local, session, ret, "fs_rename"); - goto err; - } + (void)to; /* unused */ + (void)flags; /* unused */ -err: - return (ret); + return (local_err( + WT_FS2LOCAL(file_system), session, ENOTSUP, "%s: rename of file not supported", from)); } /* * local_remove -- - * POSIX remove, for files not yet flushed to the cloud. If a file has been flushed, we don't - * support this operation. We consider anything in the cloud to be readonly as far as the custom - * file system is concerned. + * POSIX remove, not supported for cloud objects. */ static int local_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, uint32_t flags) { - LOCAL_STORAGE *local; - int ret; - bool writeable; - - (void)flags; /* Unused */ - - local = WT_FS2LOCAL(file_system); + (void)flags; /* unused */ - local->op_count++; - if ((ret = local_writeable(local, name, &writeable)) != 0) - goto err; - if (!writeable) { - ret = local_err(local, session, ENOTSUP, "%s: remove of flushed file not allowed", name); - goto err; - } - - ret = unlink(name); - if (ret != 0) { - ret = local_err(local, session, errno, "%s: ss_remove unlink", name); - goto err; - } - -err: - return (ret); + return (local_err( + WT_FS2LOCAL(file_system), session, ENOTSUP, "%s: remove of file not supported", name)); } /* @@ -1076,31 +1034,16 @@ local_size(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, w struct stat sb; LOCAL_STORAGE *local; int ret; - char *path; local = WT_FS2LOCAL(file_system); - path = NULL; - local->op_count++; + *sizep = 0; - /* If the file exists directly in the file system, it's not yet flushed, so use it */ - ret = stat(name, &sb); - if (ret == ENOENT) { - /* Otherwise, we'll see if it's in the cache directory. */ - if ((ret = local_cache_path(file_system, name, &path)) != 0) - goto err; - - ret = stat(path, &sb); - /* TODO: tiered: if we still get an ENOENT, then we'd need to ping the cloud to get the - * size. */ - } - if (ret == 0) - *sizep = sb.st_size; - else - ret = local_err(local, session, errno, "%s: ss_size stat", path); + if ((ret = local_stat(file_system, session, name, "ss_size", true, &sb)) != 0) + goto err; + *sizep = sb.st_size; err: - free(path); return (ret); } @@ -1249,14 +1192,10 @@ local_file_size(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *size static int local_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) { - LOCAL_FILE_HANDLE *local_fh; - WT_FILE_HANDLE *wt_fh; - - local_fh = (LOCAL_FILE_HANDLE *)file_handle; - wt_fh = local_fh->fh; - - local_fh->local->fh_ops++; - return (wt_fh->fh_sync(wt_fh, session)); + /* This is a no-op. We could also disallow it. */ + (void)file_handle; + (void)session; + return (0); } /* @@ -1268,13 +1207,14 @@ local_file_write( WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, const void *buf) { LOCAL_FILE_HANDLE *local_fh; - WT_FILE_HANDLE *wt_fh; - local_fh = (LOCAL_FILE_HANDLE *)file_handle; - wt_fh = local_fh->fh; + (void)offset; + (void)len; + (void)buf; - local_fh->local->write_ops++; - return (wt_fh->fh_write(wt_fh, session, offset, len, buf)); + local_fh = (LOCAL_FILE_HANDLE *)file_handle; + return (local_err(local_fh->local, session, ENOTSUP, "ss_open_object: write not supported: %s", + local_fh->iface.name)); } /* diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 9e2b467faaf..f76b0efbdf6 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "03eb45ea7fd94d6f241f8668a58d0119c39bbf52" + "commit": "27d10c3362673e12fb53b9c4be22cbb9fd46c72c" } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 32a6172cde6..ff4c436a6e1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -522,8 +522,8 @@ __cursor_key_order_check_row(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo __wt_msg(session, "WT_CURSOR.%s out-of-order returns: returned key %.1024s then key %.1024s", next ? "next" : "prev", __wt_buf_set_printable_format( - session, cbt->lastkey->data, cbt->lastkey->size, btree->key_format, a), - __wt_buf_set_printable_format(session, key->data, key->size, btree->key_format, b))); + session, cbt->lastkey->data, cbt->lastkey->size, btree->key_format, false, a), + __wt_buf_set_printable_format(session, key->data, key->size, btree->key_format, false, b))); WT_ERR(__wt_msg(session, "dumping the cursor page")); WT_ERR(__wt_debug_cursor_page(&cbt->iface, NULL)); WT_ERR_PANIC(session, EINVAL, "found key out-of-order returns"); diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index ef8e42f97e5..c952272bb8b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -944,7 +944,7 @@ __curfile_update_check(WT_CURSOR_BTREE *cbt) else if (btree->type != BTREE_COL_VAR) return (0); - return (__wt_txn_update_check(session, cbt, upd, NULL)); + return (__wt_txn_modify_check(session, cbt, upd, NULL)); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 81f419184f5..d718239a575 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -146,7 +146,7 @@ __debug_item_value(WT_DBG *ds, const char *tag, const void *data_arg, size_t siz if (session->dump_raw) return (ds->f(ds, "\t%s%s{%s}\n", tag == NULL ? "" : tag, tag == NULL ? "" : " ", - __wt_buf_set_printable(session, data_arg, size, ds->t1))); + __wt_buf_set_printable(session, data_arg, size, false, ds->t1))); /* * If the format is 'S', it's a string and our version of it may not yet be nul-terminated. @@ -157,7 +157,7 @@ __debug_item_value(WT_DBG *ds, const char *tag, const void *data_arg, size_t siz size = ds->t2->size + 1; } return (ds->f(ds, "\t%s%s{%s}\n", tag == NULL ? "" : tag, tag == NULL ? "" : " ", - __wt_buf_set_printable_format(session, data_arg, size, ds->value_format, ds->t1))); + __wt_buf_set_printable_format(session, data_arg, size, ds->value_format, false, ds->t1))); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_misc.c b/src/third_party/wiredtiger/src/btree/bt_misc.c index 6488ac7148a..9d60ec73ce8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_misc.c +++ b/src/third_party/wiredtiger/src/btree/bt_misc.c @@ -91,7 +91,7 @@ __wt_key_string( #ifdef HAVE_DIAGNOSTIC if (session->dump_raw) - return (__wt_buf_set_printable(session, data_arg, size, buf)); + return (__wt_buf_set_printable(session, data_arg, size, false, buf)); #endif /* @@ -107,7 +107,7 @@ __wt_key_string( size = sizeof(WT_ERR_STRING); } } - return (__wt_buf_set_printable_format(session, data_arg, size, key_format, buf)); + return (__wt_buf_set_printable_format(session, data_arg, size, key_format, false, buf)); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 8cb00cbde81..cc8c8d47803 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -646,10 +646,11 @@ __slvg_trk_leaf(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, uint8_t *ad __wt_verbose(session, WT_VERB_SALVAGE, "%s start key %s", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, ss->tmp1), - __wt_buf_set_printable(session, trk->row_start.data, trk->row_start.size, ss->tmp2)); + __wt_buf_set_printable( + session, trk->row_start.data, trk->row_start.size, false, ss->tmp2)); __wt_verbose(session, WT_VERB_SALVAGE, "%s stop key %s", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, ss->tmp1), - __wt_buf_set_printable(session, trk->row_stop.data, trk->row_stop.size, ss->tmp2)); + __wt_buf_set_printable(session, trk->row_stop.data, trk->row_stop.size, false, ss->tmp2)); /* Row-store pages can contain overflow items. */ WT_ERR(__slvg_trk_leaf_ovfl(session, dsk, trk)); @@ -1879,7 +1880,7 @@ __slvg_row_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref, WT_S break; __wt_verbose(session, WT_VERB_SALVAGE, "%s merge discarding leading key %s", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, ss->tmp1), - __wt_buf_set_printable(session, key->data, key->size, ss->tmp2)); + __wt_buf_set_printable(session, key->data, key->size, false, ss->tmp2)); ++skip_start; } if (F_ISSET(trk, WT_TRACK_CHECK_STOP)) @@ -1894,7 +1895,7 @@ __slvg_row_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref, WT_S break; __wt_verbose(session, WT_VERB_SALVAGE, "%s merge discarding trailing key %s", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, ss->tmp1), - __wt_buf_set_printable(session, key->data, key->size, ss->tmp2)); + __wt_buf_set_printable(session, key->data, key->size, false, ss->tmp2)); ++skip_stop; } diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 764cec570c2..27d1d90b6d7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -625,8 +625,8 @@ __verify_row_int_key_order( " on the page at %s sorts before the last key appearing on page %s, earlier in the tree: " "%s, %s", entry, __verify_addr_string(session, ref, vs->tmp1), (char *)vs->max_addr->data, - __wt_buf_set_printable(session, item.data, item.size, vs->tmp2), - __wt_buf_set_printable(session, vs->max_key->data, vs->max_key->size, vs->tmp3)); + __wt_buf_set_printable(session, item.data, item.size, false, vs->tmp2), + __wt_buf_set_printable(session, vs->max_key->data, vs->max_key->size, false, vs->tmp3)); /* Update the largest key we've seen to the key just checked. */ WT_RET(__wt_buf_set(session, vs->max_key, item.data, item.size)); @@ -677,8 +677,9 @@ __verify_row_leaf_key_order(WT_SESSION_IMPL *session, WT_REF *ref, WT_VSTUFF *vs "the first key on the page at %s sorts equal to or less than the last key appearing " "on the page at %s, earlier in the tree: %s, %s", __verify_addr_string(session, ref, vs->tmp2), (char *)vs->max_addr->data, - __wt_buf_set_printable(session, vs->tmp1->data, vs->tmp1->size, vs->tmp3), - __wt_buf_set_printable(session, vs->max_key->data, vs->max_key->size, vs->tmp4)); + __wt_buf_set_printable(session, vs->tmp1->data, vs->tmp1->size, false, vs->tmp3), + __wt_buf_set_printable( + session, vs->max_key->data, vs->max_key->size, false, vs->tmp4)); } /* Update the largest key we've seen to the last key on this page. */ @@ -804,7 +805,7 @@ __verify_key_hs( WT_ERR_MSG(session, WT_ERROR, "key %s has a overlap of timestamp ranges between history store stop timestamp %s " "being newer than a more recent timestamp range having start timestamp %s", - __wt_buf_set_printable(session, tmp1->data, tmp1->size, vs->tmp2), + __wt_buf_set_printable(session, tmp1->data, tmp1->size, false, vs->tmp2), __wt_timestamp_to_string(older_stop_ts, ts_string[0]), __wt_timestamp_to_string(newer_start_ts, ts_string[1])); } diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index 994e35806f2..d25699df070 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -249,8 +249,9 @@ __verify_row_key_order_check(WT_SESSION_IMPL *session, WT_ITEM *last, uint32_t l ret = WT_ERROR; WT_ERR_VRFY(session, "the %" PRIu32 " and %" PRIu32 " keys on page at %s are incorrectly sorted: %s, %s", - last_cell_num, cell_num, tag, __wt_buf_set_printable(session, last->data, last->size, tmp1), - __wt_buf_set_printable(session, current->data, current->size, tmp2)); + last_cell_num, cell_num, tag, + __wt_buf_set_printable(session, last->data, last->size, false, tmp1), + __wt_buf_set_printable(session, current->data, current->size, false, tmp2)); err: __wt_scr_free(session, &tmp1); diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index 632526f83df..c14d3251042 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -35,15 +35,16 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; u_int i, skipdepth; - bool append, logged; + bool append, inserted_to_update_chain, logged; btree = CUR2BT(cbt); ins = NULL; page = cbt->ref->page; session = CUR2S(cbt); + last_upd = NULL; upd = upd_arg; prev_upd_ts = WT_TS_NONE; - append = logged = false; + append = inserted_to_update_chain = logged = false; /* * We should have one of the following: @@ -120,19 +121,15 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U } /* - * Delete, insert or update a column-store entry. - * - * If modifying a previously modified record, cursor.ins will be set to point to the correct - * update list. Create a new update entry and link it into the existing list. - * - * Else, allocate an insert array as necessary, build an insert/update structure pair, and link - * it into place. + * Modify a column-store entry. If modifying a previously modified record, cursor.ins will point + * to the correct update list; create a new update and link it into the already existing list. + * Otherwise, we have to insert a new insert/update pair into the column-store insert list. */ if (cbt->compare == 0 && cbt->ins != NULL) { old_upd = cbt->ins->upd; if (upd_arg == NULL) { - /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd, &prev_upd_ts)); + /* Make sure the modify can proceed. */ + WT_ERR(__wt_txn_modify_check(session, cbt, old_upd, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); @@ -142,7 +139,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_ERR(__wt_txn_modify(session, upd)); logged = true; - /* Avoid a data copy in WT_CURSOR.update. */ + /* Avoid WT_CURSOR.update data copy. */ __wt_upd_value_assign(cbt->modify_update, upd); } else { upd_size = __wt_update_list_memsize(upd); @@ -178,6 +175,10 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U /* Serialize the update. */ WT_ERR(__wt_update_serial(session, cbt, page, &cbt->ins->upd, &upd, upd_size, false)); } else { + /* Make sure the modify can proceed. */ + if (cbt->compare == 0 && upd_arg == NULL) + WT_ERR(__wt_txn_modify_check(session, cbt, NULL, NULL)); + /* Allocate the append/update list reference as necessary. */ if (append) { WT_PAGE_ALLOC_AND_SWAP(session, page, mod->mod_col_append, ins_headp, 1); @@ -218,7 +219,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_ERR(__wt_txn_modify(session, upd)); logged = true; - /* Avoid a data copy in WT_CURSOR.update. */ + /* Avoid WT_CURSOR.update data copy. */ __wt_upd_value_assign(cbt->modify_update, upd); } else upd_size = __wt_update_list_memsize(upd); @@ -254,12 +255,14 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U session, page, cbt->ins_head, cbt->ins_stack, &ins, ins_size, skipdepth, exclusive)); } + inserted_to_update_chain = true; + /* If the update was successful, add it to the in-memory log. */ if (logged && modify_type != WT_UPDATE_RESERVE) { WT_ERR(__wt_txn_log_op(session, cbt)); /* - * In case of append, the recno (key) for the value is assigned now. Set the recno in the + * In case of append, the recno (key) for the value is assigned now. Set the key in the * transaction operation to be used in case this transaction is prepared to retrieve the * update corresponding to this operation. */ @@ -268,14 +271,25 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U if (0) { err: - /* - * Remove the update from the current transaction, so we don't try to modify it on rollback. - */ + /* Remove the update from the current transaction; don't try to modify it on rollback. */ if (logged) __wt_txn_unmodify(session); + + /* Free any allocated insert list object. */ __wt_free(session, ins); - if (upd_arg == NULL) + + cbt->ins = NULL; + + /* Discard any allocated update, unless we failed after linking it into page memory. */ + if (upd_arg == NULL && !inserted_to_update_chain) __wt_free(session, upd); + + /* + * When prepending a list of updates to an update chain, we link them together; sever that + * link so our callers list doesn't point into page memory. + */ + if (last_upd != NULL) + last_upd->next = NULL; } return (ret); diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index c96b963cb1f..779f5112c12 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -109,8 +109,8 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, upd_entry = &cbt->ins->upd; if (upd_arg == NULL) { - /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd = *upd_entry, &prev_upd_ts)); + /* Make sure the modify can proceed. */ + WT_ERR(__wt_txn_modify_check(session, cbt, old_upd = *upd_entry, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); @@ -139,6 +139,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, upd_arg->next == NULL) || (upd_arg->type == WT_UPDATE_TOMBSTONE && upd_arg->next != NULL && upd_arg->next->type == WT_UPDATE_STANDARD && upd_arg->next->next == NULL)); + upd_size = __wt_update_list_memsize(upd); /* If there are existing updates, append them after the new updates. */ @@ -182,7 +183,6 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, * slot. That's hard, so we set a flag. */ WT_PAGE_ALLOC_AND_SWAP(session, page, mod->mod_row_insert, ins_headp, page->entries + 1); - ins_slot = F_ISSET(cbt, WT_CBT_SEARCH_SMALLEST) ? page->entries : cbt->slot; ins_headp = &mod->mod_row_insert[ins_slot]; @@ -218,6 +218,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, (upd_arg->type == WT_UPDATE_TOMBSTONE && upd_arg->next != NULL && upd_arg->next->type == WT_UPDATE_STANDARD && upd_arg->next->next == NULL) || (upd_arg->type == WT_UPDATE_STANDARD && upd_arg->next == NULL)); + upd_size = __wt_update_list_memsize(upd); } @@ -250,8 +251,10 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, inserted_to_update_chain = true; + /* If the update was successful, add it to the in-memory log. */ if (logged && modify_type != WT_UPDATE_RESERVE) { WT_ERR(__wt_txn_log_op(session, cbt)); + /* * Set the key in the transaction operation to be used in case this transaction is prepared * to retrieve the update corresponding to this operation. @@ -261,15 +264,23 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, if (0) { err: - /* - * Remove the update from the current transaction, so we don't try to modify it on rollback. - */ + /* Remove the update from the current transaction, don't try to modify it on rollback. */ if (logged) __wt_txn_unmodify(session); + + /* Free any allocated insert list object. */ __wt_free(session, ins); + cbt->ins = NULL; + + /* Discard any allocated update, unless we failed after linking it into page memory. */ if (upd_arg == NULL && !inserted_to_update_chain) __wt_free(session, upd); + + /* + * When prepending a list of updates to an update chain, we link them together; sever that + * link so our callers list doesn't point into page memory. + */ if (last_upd != NULL) last_upd->next = NULL; } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 693bf9aa554..a3d9af315a4 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -359,7 +359,10 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = { {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_wait", "boolean", NULL, NULL, NULL, 0}, {"debug", "category", NULL, NULL, confchk_WT_SESSION_open_cursor_debug_subconfigs, 1}, - {"dump", "string", NULL, "choices=[\"hex\",\"json\",\"pretty\",\"print\"]", NULL, 0}, + {"dump", "string", NULL, + "choices=[\"hex\",\"json\",\"pretty\",\"pretty_hex\"," + "\"print\"]", + NULL, 0}, {"incremental", "category", NULL, NULL, confchk_WT_SESSION_open_cursor_incremental_subconfigs, 7}, {"next_random", "boolean", NULL, NULL, NULL, 0}, {"next_random_sample_size", "string", NULL, NULL, NULL, 0}, diff --git a/src/third_party/wiredtiger/src/cursor/cur_bulk.c b/src/third_party/wiredtiger/src/cursor/cur_bulk.c index c23c1940cd6..5672cbbf1c7 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_bulk.c +++ b/src/third_party/wiredtiger/src/cursor/cur_bulk.c @@ -202,8 +202,8 @@ __bulk_row_keycmp_err(WT_CURSOR_BULK *cbulk) WT_ERR_MSG(session, EINVAL, "bulk-load presented with out-of-order keys: %s compares smaller than previously inserted " "key %s", - __wt_buf_set_printable(session, cursor->key.data, cursor->key.size, a), - __wt_buf_set_printable(session, cbulk->last.data, cbulk->last.size, b)); + __wt_buf_set_printable(session, cursor->key.data, cursor->key.size, false, a), + __wt_buf_set_printable(session, cbulk->last.data, cbulk->last.size, false, b)); err: __wt_scr_free(session, &a); diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 734acdbe70a..be4c1c7ef34 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -91,8 +91,8 @@ __curdump_get_key(WT_CURSOR *cursor, ...) WT_ERR(child->get_key(child, &item)); if (F_ISSET(cursor, WT_CURSTD_DUMP_PRETTY)) { - WT_IGNORE_RET_PTR(__wt_buf_set_printable_format( - session, item.data, item.size, cursor->key_format, &cursor->key)); + WT_IGNORE_RET_PTR(__wt_buf_set_printable_format(session, item.data, item.size, + cursor->key_format, F_ISSET(cursor, WT_CURSTD_DUMP_HEX), &cursor->key)); } else WT_ERR( __raw_to_dump(session, &item, &cursor->key, F_ISSET(cursor, WT_CURSTD_DUMP_HEX))); @@ -241,8 +241,8 @@ __curdump_get_value(WT_CURSOR *cursor, ...) WT_ERR(child->get_value(child, &item)); if (F_ISSET(cursor, WT_CURSTD_DUMP_PRETTY)) - WT_IGNORE_RET_PTR(__wt_buf_set_printable_format( - session, item.data, item.size, cursor->value_format, &cursor->value)); + WT_IGNORE_RET_PTR(__wt_buf_set_printable_format(session, item.data, item.size, + cursor->value_format, F_ISSET(cursor, WT_CURSTD_DUMP_HEX), &cursor->value)); else WT_ERR( __raw_to_dump(session, &item, &cursor->value, F_ISSET(cursor, WT_CURSTD_DUMP_HEX))); diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 26d6915724a..adacc1f3e68 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -1189,6 +1189,8 @@ __wt_cursor_init( dump_flag = WT_CURSTD_DUMP_PRINT; else if (WT_STRING_MATCH("pretty", cval.str, cval.len)) dump_flag = WT_CURSTD_DUMP_PRETTY; + else if (WT_STRING_MATCH("pretty_hex", cval.str, cval.len)) + dump_flag = WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRETTY; else dump_flag = WT_CURSTD_DUMP_HEX; F_SET(cursor, dump_flag); diff --git a/src/third_party/wiredtiger/src/docs/arch-hs.dox b/src/third_party/wiredtiger/src/docs/arch-hs.dox index e1d0a88c677..c6453108f03 100644 --- a/src/third_party/wiredtiger/src/docs/arch-hs.dox +++ b/src/third_party/wiredtiger/src/docs/arch-hs.dox @@ -89,7 +89,7 @@ be corrected. Consider the following update chain for a user table with btree id 1000 and data store key "AAA": -@plantuml_start{hs_update_chain.png} +@plantuml_start{hs_update_chain.png } @startuml{hs_update_chain.png} together { diff --git a/src/third_party/wiredtiger/src/docs/arch-log-file.dox b/src/third_party/wiredtiger/src/docs/arch-log-file.dox index a684e023d76..1ff828052db 100644 --- a/src/third_party/wiredtiger/src/docs/arch-log-file.dox +++ b/src/third_party/wiredtiger/src/docs/arch-log-file.dox @@ -1,7 +1,59 @@ /*! @arch_page arch-log-file Log File Format -The format of a WiredTiger log file is defined in \c log.h . -Each file begins with a fixed length header, followed by a set of -variable length log records. Though the records may have varying -length, they all begin on boundaries aligned to 128 bytes. +This page assumes familiarity with the logging subsystem. Please see +@ref arch-logging for the logging subsystem architecture description. + +Each log file begins with a fixed length header, followed by a set of +variable length log records. The fixed length header contains information +necessary for WiredTiger to determine that a file is a valid log file. +The header contains a magic number in the first 32 bits, the log version +number and the configured maximum log file size. + +When a new log file is created WiredTiger must synchronously write out the header before allowing +any user records to be written to the file. The content of the file header is +identical for every log file. This fact allows for pre-allocation of log files +so that user threads that happen to be writing out log records that require a log +file switch are usually not penalized with expensive I/O operations. The typical +log file switch is a rename instead. In addition to writing out the header, pre-allocation +also allocates the space for the maximum size of the log file so that the risk +of out of space errors is minimized. Pre-allocation is done with an internal +thread as described in @ref log_threads. + +Log files from log version 1 are generated by releases before WiredTiger version +3.0.0. In log version 1, user records start immediately after the header. +All log files from log version 2 onward write a system log record after +the header as the first record in the log file. Writing the system record +is part of creating the new log file. + +All records in the log, including the header, are subject to the minimum record +size. Typically that is 128 bytes. That value may be changed if direct I/O for +logging is turned on. When enabling direct I/O via \c direct_io=log to ::wiredtiger_open +then the minimum alignment is the value specified for \c buffer_alignment. +In all other cases it is 128 bytes. + +The reason that 128 bytes was chosen as the minimum record size was to eliminate any +possibility of false sharing in memory and invalidating cache lines when multiple +threads are writing their log records to the memory buffers in parallel. False sharing +would potentially impact performance. + +From version 2 onward the first record in the log file, typically at offset 128 in the +file, is a special system log +record that contains the LSN of the end of the previous log file. For the first +log file that value is an invalid LSN. For later log files the LSN indicates the end +of the previous file so that recovery can detect missing log records and holes in +log files that appear at the end of a file. + +As described in @ref log_subsystem, multiple threads are writing records to the log +in a lock free manner and multiple log buffer slots may be in flight at any given time. +A hole can be generated in a log file if a buffer with a later LSN is written before +a buffer with an earlier LSN. That can also happen at a log file boundary and that is why +knowing the LSN at the end of the previous log file is critical to recovery. + +The user can choose the maximum log file size via \c log=(file_max=size) configuration +to the ::wiredtiger_open call. Records written in the log are varying length +depending on the data written. In typical usage, the system will choose to switch +log files before writing a log buffer that exceeds the configured file size. However it +is possible that a single record itself could be larger than the configured file size. +In that case the system must allow that record to exceed the maximum file size. But it +will be the only record in that log file. */ diff --git a/src/third_party/wiredtiger/src/docs/build-posix.dox b/src/third_party/wiredtiger/src/docs/build-posix.dox index eb3528e249a..ea9bcb5ea90 100644 --- a/src/third_party/wiredtiger/src/docs/build-posix.dox +++ b/src/third_party/wiredtiger/src/docs/build-posix.dox @@ -141,19 +141,19 @@ configures WiredTiger to use gcc-based spinlocks). @section posix_compiler Changing compiler or loader options By default CMake will use your default system compiler (\c cc). If you want to use a specific toolchain you can pass a toolchain file. We have -provided a toolchain file for both GCC (\c build_cmake/toolchains/gcc.cmake) and Clang (\c build_cmake/toolchains/clang.cmake). To use either +provided a toolchain file for both GCC (\c cmake/toolchains/gcc.cmake) and Clang (\c cmake/toolchains/clang.cmake). To use either toolchain you can pass the \c -DCMAKE_TOOLCHAIN_FILE= to the CMake configuration step. For example: For example, to explicitly specify the GCC toolchain: @code -cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/gcc.cmake -G Ninja ../. +cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/gcc.cmake -G Ninja ../. @endcode To explicitly specify the Clang toolchain: @code -cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/clang.cmake -G Ninja ../. +cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -G Ninja ../. @endcode By default, WiredTiger builds with the \c -O3 compiler optimization flag diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index 4432ae32054..7d518c4a18e 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -187,7 +187,8 @@ format. @par <code>-p</code> Dump in human-readable format (pretty-print). The \c -p flag is incompatible -with the \c load command. +with the \c load command. The \c -p flag can be combined with \c -x. In this case, raw data elements +will be formatted like \c -x with hexadecimal encoding. @par <code>-r</code> Dump in reverse order, from largest key to smallest. @@ -199,7 +200,10 @@ timestamp. @par <code>-x</code> Dump all characters in a hexadecimal encoding (the default is to leave -printable characters unencoded). +printable characters unencoded). The \c -x flag can be combined with \c -p. In this case, the dump +will be formatted similar to \c -p except for raw data elements, which will look like \c -x with +hexadecimal encoding. If the two options are combined the output is no longer compatible +with \c load. <hr> @section util_list wt list diff --git a/src/third_party/wiredtiger/src/docs/dump-formats.dox b/src/third_party/wiredtiger/src/docs/dump-formats.dox index 2a903f9c790..2a5a6475646 100644 --- a/src/third_party/wiredtiger/src/docs/dump-formats.dox +++ b/src/third_party/wiredtiger/src/docs/dump-formats.dox @@ -68,6 +68,7 @@ following information: @hrow{String, Meaning} @row{hex, the dumped data is in a hexadecimal dump format} @row{print, the dumped data is in a printable format} +@row{print hex, the dumped data is in a printable format with raw byte array elements printed same as in hex format} </table> The dump header follows a single \c "Header" line in the file and @@ -77,10 +78,11 @@ string. The table or file can be recreated by calling WT_SESSION::create for each pair of lines in the header. The dump body follows a single \c "Data" line in the file and consists -of a text representation of the records in the table. Each record is a +of a text representation of the records in the table. Each record is represented by a pair of lines: the first line is the key and the second -line is the value. These lines are encoded in one of two formats: a -printable format and a hexadecimal format. +line is the value. These lines are encoded in one of the following formats: a +printable format, a hexadecimal format, and a printable format with raw byte array elements in +hexadecimal format. The printable format consists of literal printable characters, and hexadecimal encoded non-printable characters. Encoded characters are @@ -97,6 +99,9 @@ literal character is written as a pair of characters (first the high-nibble and then the low-nibble). For example, "0a" would be an ASCII newline character and "1b" would be an ASCII escape character. +The printable hexadecimal format is the same as the printable except all raw +byte array elements will be printed in hexadecimal format. + Because the definition of "printable" may depend on the application's locale, dump files in the printable output format may be less portable than dump files in the hexadecimal output format. diff --git a/src/third_party/wiredtiger/src/docs/schema.dox b/src/third_party/wiredtiger/src/docs/schema.dox index 57f2801884f..036411360c3 100644 --- a/src/third_party/wiredtiger/src/docs/schema.dox +++ b/src/third_party/wiredtiger/src/docs/schema.dox @@ -67,7 +67,7 @@ struct module to describe the types of columns in a table: <table> @hrow{Format, C Type, Python type, Notes} -@row{\c x, N/A, N/A, N/A, pad byte\, no associated value} +@row{\c x, N/A, N/A, pad byte\, no associated value} @row{\c b, \c int8_t, \c int, signed byte} @row{\c B, \c uint8_t, \c int, unsigned byte} @row{\c h, \c int16_t, \c int, signed 16-bit} diff --git a/src/third_party/wiredtiger/src/docs/transactions.dox b/src/third_party/wiredtiger/src/docs/transactions.dox index c594b6b15c9..3c1233b0400 100644 --- a/src/third_party/wiredtiger/src/docs/transactions.dox +++ b/src/third_party/wiredtiger/src/docs/transactions.dox @@ -246,6 +246,15 @@ For example, for the timestamp values <code>prepare_timestamp=100, timestamp is rounded up to the new prepare timestamp as <code>commit_timestamp=200</code>. +Configuring <code>roundup_timestamps=(prepared=true)</code> also allows setting +the prepared timestamps smaller than or equal to the latest active read timestamps. +Use this feature carefully as it may break repeated read. For example, consider a +transaction with a read timestamp set to 30 and a key that has a value with +timestamp 20. Before a prepared transaction (in another thread), reading the key +returns the value. Now a prepared transaction elsewhere in the system sets a prepared +timestamp at 30. The reader, trying to do a repeat read of the key, gets return a +prepared conflict error instead of the value. + Configuring <code>roundup_timestamps=(read=true)</code> causes the read timestamp to be rounded up to the oldest timestamp, if the read timestamp is greater than the oldest timestamp no change will be made. diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index c5f4b2d013e..e946fa2e2eb 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -666,7 +666,7 @@ select adaptive pthread mutexes by specifying <dt>LSM merge threads option change</dt> <dd> The WT_SESSION::create \c lsm=(merge_threads) configuration option has been -replaced by the W::wiredtiger_open \c lsm_manager=(worker_thread_max) option. +replaced by the ::wiredtiger_open \c lsm_manager=(worker_thread_max) option. The new version specifies a set of LSM threads that are shared across all LSM trees in a database, the older configuration was per LSM table. </dd> diff --git a/src/third_party/wiredtiger/src/history/hs_verify.c b/src/third_party/wiredtiger/src/history/hs_verify.c index 3b73a587922..986e89fdb2b 100644 --- a/src/third_party/wiredtiger/src/history/hs_verify.c +++ b/src/third_party/wiredtiger/src/history/hs_verify.c @@ -75,7 +75,7 @@ __hs_verify_id( F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); WT_ERR_PANIC(session, WT_PANIC, "the associated history store key %s was not found in the data store %s", - __wt_buf_set_printable(session, key.data, key.size, prev_key), + __wt_buf_set_printable(session, key.data, key.size, false, prev_key), session->dhandle->name); } @@ -181,7 +181,7 @@ __wt_hs_verify(WT_SESSION_IMPL *session) F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); WT_ERR_PANIC(session, WT_PANIC, "Unable to find btree id %" PRIu32 " in the metadata file for the associated key %s", - btree_id, __wt_buf_set_printable(session, key.data, key.size, buf)); + btree_id, __wt_buf_set_printable(session, key.data, key.size, false, buf)); } WT_ERR(__wt_open_cursor(session, uri_data, NULL, NULL, &ds_cursor)); F_SET(ds_cursor, WT_CURSOR_RAW_OK); diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 68b89d99b0e..1892edd09a3 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -54,9 +54,10 @@ extern const WT_CONFIG_ENTRY *__wt_test_config_match(const char *test_name) extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_buf_set_printable(WT_SESSION_IMPL *session, const void *p, size_t size, - WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + bool hexonly, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_buf_set_printable_format(WT_SESSION_IMPL *session, const void *buffer, - size_t size, const char *format, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + size_t size, const char *format, bool hexonly, WT_ITEM *buf) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_buf_set_size(WT_SESSION_IMPL *session, uint64_t size, bool exact, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_cell_type_string(uint8_t type) @@ -2134,6 +2135,8 @@ static inline int __wt_txn_idle_cache_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_txn_modify_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, + WT_UPDATE *upd, wt_timestamp_t *prev_tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_modify_page_delete(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_op_set_key(WT_SESSION_IMPL *session, const WT_ITEM *key) @@ -2147,8 +2150,6 @@ static inline int __wt_txn_read_upd_list_internal(WT_SESSION_IMPL *session, WT_C WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_search_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_UPDATE *upd, wt_timestamp_t *prev_tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, u_int modify_type, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp, diff --git a/src/third_party/wiredtiger/src/include/serial_inline.h b/src/third_party/wiredtiger/src/include/serial_inline.h index 8e6813f2866..4f6384732dc 100644 --- a/src/third_party/wiredtiger/src/include/serial_inline.h +++ b/src/third_party/wiredtiger/src/include/serial_inline.h @@ -242,7 +242,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page * Check if our update is still permitted. */ while (!__wt_atomic_cas_ptr(srch_upd, upd->next, upd)) { - if ((ret = __wt_txn_update_check(session, cbt, upd->next = *srch_upd, &prev_upd_ts)) != 0) { + if ((ret = __wt_txn_modify_check(session, cbt, upd->next = *srch_upd, &prev_upd_ts)) != 0) { /* Free unused memory on error. */ __wt_free(session, upd); return (ret); diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 893179db355..e590bae7f20 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -743,6 +743,8 @@ struct __wt_connection_stats { int64_t txn_rts_pages_visited; int64_t txn_rts_hs_restore_tombstones; int64_t txn_rts_hs_restore_updates; + int64_t txn_rts_delete_rle_skipped; + int64_t txn_rts_stable_rle_skipped; int64_t txn_rts_sweep_hs_keys; int64_t txn_rts_tree_walk_skip_pages; int64_t txn_rts_upd_aborted; @@ -1014,6 +1016,8 @@ struct __wt_dsrc_stats { int64_t txn_rts_keys_restored; int64_t txn_rts_hs_restore_tombstones; int64_t txn_rts_hs_restore_updates; + int64_t txn_rts_delete_rle_skipped; + int64_t txn_rts_stable_rle_skipped; int64_t txn_rts_sweep_hs_keys; int64_t txn_rts_hs_removed; int64_t txn_checkpoint_obsolete_applied; diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index df81b29fe23..6d3b4e8fa55 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -1304,11 +1304,11 @@ __wt_txn_search_check(WT_SESSION_IMPL *session) } /* - * __wt_txn_update_check -- - * Check if the current transaction can update an item. + * __wt_txn_modify_check -- + * Check if the current transaction can modify an item. */ static inline int -__wt_txn_update_check( +__wt_txn_modify_check( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, wt_timestamp_t *prev_tsp) { WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 2da0d297fb1..e292d1c74fd 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -902,10 +902,12 @@ struct __wt_session { * @config{dump, configure the cursor for dump format inputs and outputs: "hex" selects a * simple hexadecimal format\, "json" selects a JSON format with each record formatted as * fields named by column names if available\, "pretty" selects a human-readable format - * (making it incompatible with the "load") and "print" selects a format where only - * non-printing characters are hexadecimal encoded. These formats are compatible with the - * @ref util_dump and @ref util_load commands., a string\, chosen from the following - * options: \c "hex"\, \c "json"\, \c "pretty"\, \c "print"; default empty.} + * (making it incompatible with the "load")\, "pretty_hex" is similar to "pretty" (also + * incompatible with "load") except raw byte data elements will be printed like "hex" + * format\, and "print" selects a format where only non-printing characters are hexadecimal + * encoded. These formats are compatible with the @ref util_dump and @ref util_load + * commands., a string\, chosen from the following options: \c "hex"\, \c "json"\, \c + * "pretty"\, \c "pretty_hex"\, \c "print"; default empty.} * @config{incremental = (, configure the cursor for block incremental backup usage. These * formats are only compatible with the backup data source; see @ref backup., a set of * related configuration options defined below.} @@ -1675,11 +1677,12 @@ struct __wt_session { * and the commit timestamp of this transaction can be rounded up. If the prepare timestamp * is less than the oldest timestamp\, the prepare timestamp will be rounded to the oldest * timestamp. If the commit timestamp is less than the prepare timestamp\, the commit - * timestamp will be rounded up to the prepare timestamp., a boolean flag; default \c - * false.} - * @config{ read, if the read timestamp is less than the - * oldest timestamp\, the read timestamp will be rounded up to the oldest timestamp., a - * boolean flag; default \c false.} + * timestamp will be rounded up to the prepare timestamp. Allows setting the prepared + * timestamp smaller than or equal to the latest active read timestamp., a boolean flag; + * default \c false.} + * @config{ read, if the read timestamp is less + * than the oldest timestamp\, the read timestamp will be rounded up to the oldest + * timestamp., a boolean flag; default \c false.} * @config{ ),,} * @config{sync, whether to sync log records when the transaction commits\, inherited from * ::wiredtiger_open \c transaction_sync., a boolean flag; default empty.} @@ -6008,140 +6011,144 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1428 /*! transaction: rollback to stable restored updates from history store */ #define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1429 +/*! transaction: rollback to stable skipping delete rle */ +#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1430 +/*! transaction: rollback to stable skipping stable rle */ +#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1431 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1430 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1432 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1431 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1433 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1432 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1434 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1433 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1435 /*! transaction: sessions scanned in each walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1434 +#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1436 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1435 +#define WT_STAT_CONN_TXN_SET_TS 1437 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1436 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1438 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1437 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1439 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1438 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1440 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1439 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1441 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1440 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1442 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1441 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1443 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1442 +#define WT_STAT_CONN_TXN_BEGIN 1444 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1443 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1445 /*! * transaction: transaction checkpoint currently running for history * store file */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1444 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1446 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1445 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1447 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1446 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1448 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1447 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1449 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1448 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1450 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1449 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1451 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1450 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1452 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1451 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1453 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1452 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1454 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1453 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1455 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1454 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1456 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1455 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1457 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1456 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1458 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1457 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1459 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1458 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1460 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1461 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1462 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1461 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1463 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1462 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1464 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1463 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1465 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1464 +#define WT_STAT_CONN_TXN_CHECKPOINT 1466 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1465 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1467 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1466 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1468 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1467 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1469 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1468 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1470 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1469 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1471 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1470 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1472 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1471 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1473 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1472 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1474 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1473 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1475 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1474 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1476 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1475 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1477 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1476 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1478 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1477 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1479 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1478 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1480 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1479 +#define WT_STAT_CONN_TXN_COMMIT 1481 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1480 +#define WT_STAT_CONN_TXN_ROLLBACK 1482 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1481 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1483 /*! * @} @@ -6772,14 +6779,18 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2210 /*! transaction: rollback to stable restored updates from history store */ #define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2211 +/*! transaction: rollback to stable skipping delete rle */ +#define WT_STAT_DSRC_TXN_RTS_DELETE_RLE_SKIPPED 2212 +/*! transaction: rollback to stable skipping stable rle */ +#define WT_STAT_DSRC_TXN_RTS_STABLE_RLE_SKIPPED 2213 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2212 +#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2214 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2213 +#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2215 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2214 +#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2216 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2215 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2217 /*! * @} diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 9518a3b63c3..4ea57b8acc7 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -327,7 +327,9 @@ __wt_rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) WT_RET(__wt_rec_split_init(session, r, page, pageref->ref_recno, btree->maxleafpage)); /* Copy the original, disk-image bytes into place. */ - memcpy(r->first_free, page->pg_fix_bitf, __bitstr_size((size_t)page->entries * btree->bitcnt)); + if (page->entries != 0) + memcpy( + r->first_free, page->pg_fix_bitf, __bitstr_size((size_t)page->entries * btree->bitcnt)); /* Update any changes to the original on-page data items. */ WT_SKIP_FOREACH (ins, WT_COL_UPDATE_SINGLE(page)) { diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 667e8c119b7..9ad305792b9 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -2058,7 +2058,7 @@ __rec_split_dump_keys(WT_SESSION_IMPL *session, WT_RECONCILE *r) for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) __wt_verbose(session, WT_VERB_SPLIT, "starting key %s", __wt_buf_set_printable( - session, WT_IKEY_DATA(multi->key.ikey), multi->key.ikey->size, tkey)); + session, WT_IKEY_DATA(multi->key.ikey), multi->key.ikey->size, false, tkey)); __wt_scr_free(session, &tkey); } else for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 33c80a3a0ed..a6fff0b3e6c 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -117,9 +117,17 @@ err: * Set the contents of the buffer to a printable representation of a byte string. */ const char * -__wt_buf_set_printable(WT_SESSION_IMPL *session, const void *p, size_t size, WT_ITEM *buf) +__wt_buf_set_printable( + WT_SESSION_IMPL *session, const void *p, size_t size, bool hexonly, WT_ITEM *buf) { - if (__wt_raw_to_esc_hex(session, p, size, buf)) { + WT_DECL_RET; + + if (hexonly) + ret = __wt_raw_to_hex(session, p, size, buf); + else + ret = __wt_raw_to_esc_hex(session, p, size, buf); + + if (ret != 0) { buf->data = "[Error]"; buf->size = strlen("[Error]"); } @@ -132,8 +140,8 @@ __wt_buf_set_printable(WT_SESSION_IMPL *session, const void *p, size_t size, WT_ * format. */ const char * -__wt_buf_set_printable_format( - WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *format, WT_ITEM *buf) +__wt_buf_set_printable_format(WT_SESSION_IMPL *session, const void *buffer, size_t size, + const char *format, bool hexonly, WT_ITEM *buf) { WT_DECL_ITEM(tmp); WT_DECL_PACK_VALUE(pv); @@ -166,7 +174,7 @@ __wt_buf_set_printable_format( if (tmp == NULL) WT_ERR(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_buf_catfmt(session, buf, "%s%s", sep, - __wt_buf_set_printable(session, pv.u.item.data, pv.u.item.size, tmp))); + __wt_buf_set_printable(session, pv.u.item.data, pv.u.item.size, hexonly, tmp))); break; case 'b': case 'h': @@ -203,7 +211,7 @@ err: * is truncated, and then passed here by a page debugging routine). Our current callers aren't * interested in error handling in such cases, return a byte string instead. */ - return (__wt_buf_set_printable(session, buffer, size, buf)); + return (__wt_buf_set_printable(session, buffer, size, hexonly, buf)); } /* diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index bc13dd8aab7..c33d3b60bef 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -221,6 +221,8 @@ static const char *const __stats_dsrc_desc[] = { "transaction: rollback to stable keys restored", "transaction: rollback to stable restored tombstones from history store", "transaction: rollback to stable restored updates from history store", + "transaction: rollback to stable skipping delete rle", + "transaction: rollback to stable skipping stable rle", "transaction: rollback to stable sweeping history store keys", "transaction: rollback to stable updates removed from history store", "transaction: transaction checkpoints due to obsolete pages", @@ -477,6 +479,8 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->txn_rts_keys_restored = 0; stats->txn_rts_hs_restore_tombstones = 0; stats->txn_rts_hs_restore_updates = 0; + stats->txn_rts_delete_rle_skipped = 0; + stats->txn_rts_stable_rle_skipped = 0; stats->txn_rts_sweep_hs_keys = 0; stats->txn_rts_hs_removed = 0; stats->txn_checkpoint_obsolete_applied = 0; @@ -720,6 +724,8 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->txn_rts_keys_restored += from->txn_rts_keys_restored; to->txn_rts_hs_restore_tombstones += from->txn_rts_hs_restore_tombstones; to->txn_rts_hs_restore_updates += from->txn_rts_hs_restore_updates; + to->txn_rts_delete_rle_skipped += from->txn_rts_delete_rle_skipped; + to->txn_rts_stable_rle_skipped += from->txn_rts_stable_rle_skipped; to->txn_rts_sweep_hs_keys += from->txn_rts_sweep_hs_keys; to->txn_rts_hs_removed += from->txn_rts_hs_removed; to->txn_checkpoint_obsolete_applied += from->txn_checkpoint_obsolete_applied; @@ -971,6 +977,8 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->txn_rts_keys_restored += WT_STAT_READ(from, txn_rts_keys_restored); to->txn_rts_hs_restore_tombstones += WT_STAT_READ(from, txn_rts_hs_restore_tombstones); to->txn_rts_hs_restore_updates += WT_STAT_READ(from, txn_rts_hs_restore_updates); + to->txn_rts_delete_rle_skipped += WT_STAT_READ(from, txn_rts_delete_rle_skipped); + to->txn_rts_stable_rle_skipped += WT_STAT_READ(from, txn_rts_stable_rle_skipped); to->txn_rts_sweep_hs_keys += WT_STAT_READ(from, txn_rts_sweep_hs_keys); to->txn_rts_hs_removed += WT_STAT_READ(from, txn_rts_hs_removed); to->txn_checkpoint_obsolete_applied += WT_STAT_READ(from, txn_checkpoint_obsolete_applied); @@ -1419,6 +1427,8 @@ static const char *const __stats_connection_desc[] = { "transaction: rollback to stable pages visited", "transaction: rollback to stable restored tombstones from history store", "transaction: rollback to stable restored updates from history store", + "transaction: rollback to stable skipping delete rle", + "transaction: rollback to stable skipping stable rle", "transaction: rollback to stable sweeping history store keys", "transaction: rollback to stable tree walk skipping pages", "transaction: rollback to stable updates aborted", @@ -1942,6 +1952,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->txn_rts_pages_visited = 0; stats->txn_rts_hs_restore_tombstones = 0; stats->txn_rts_hs_restore_updates = 0; + stats->txn_rts_delete_rle_skipped = 0; + stats->txn_rts_stable_rle_skipped = 0; stats->txn_rts_sweep_hs_keys = 0; stats->txn_rts_tree_walk_skip_pages = 0; stats->txn_rts_upd_aborted = 0; @@ -2476,6 +2488,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->txn_rts_pages_visited += WT_STAT_READ(from, txn_rts_pages_visited); to->txn_rts_hs_restore_tombstones += WT_STAT_READ(from, txn_rts_hs_restore_tombstones); to->txn_rts_hs_restore_updates += WT_STAT_READ(from, txn_rts_hs_restore_updates); + to->txn_rts_delete_rle_skipped += WT_STAT_READ(from, txn_rts_delete_rle_skipped); + to->txn_rts_stable_rle_skipped += WT_STAT_READ(from, txn_rts_stable_rle_skipped); to->txn_rts_sweep_hs_keys += WT_STAT_READ(from, txn_rts_sweep_hs_keys); to->txn_rts_tree_walk_skip_pages += WT_STAT_READ(from, txn_rts_tree_walk_skip_pages); to->txn_rts_upd_aborted += WT_STAT_READ(from, txn_rts_upd_aborted); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_opener.c b/src/third_party/wiredtiger/src/tiered/tiered_opener.c index 0e28bcdcbcb..e65d52e4694 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_opener.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_opener.c @@ -35,15 +35,17 @@ __tiered_opener_open(WT_BLOCK_FILE_OPENER *opener, WT_SESSION_IMPL *session, uin object_name = tiered->tiers[WT_TIERED_INDEX_LOCAL].name; if (!WT_PREFIX_SKIP(object_name, "file:")) WT_RET_MSG(session, EINVAL, "expected a 'file:' URI"); + WT_ERR(__wt_open(session, object_name, type, flags, fhp)); } else { WT_ERR( __wt_tiered_name(session, &tiered->iface, object_id, WT_TIERED_NAME_OBJECT, &object_uri)); object_name = object_uri; WT_PREFIX_SKIP_REQUIRED(session, object_name, "object:"); bstorage = tiered->bstorage; + flags |= WT_FS_OPEN_READONLY; + WT_WITH_BUCKET_STORAGE( + bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); } - WT_WITH_BUCKET_STORAGE( - bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); err: __wt_free(session, object_uri); return (ret); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 68e0e45a8ea..35eb2e08d6f 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -723,12 +723,14 @@ __wt_txn_release(WT_SESSION_IMPL *session) } /* - * __txn_append_hs_record -- - * Append the update older than the prepared update to the update chain + * __txn_locate_hs_record -- + * Locate the update older than the prepared update in the history store and append it to the + * update chain if necessary. */ static int -__txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_PAGE *page, - WT_UPDATE *chain, bool commit, WT_UPDATE **fix_updp, bool *upd_appended) +__txn_locate_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_PAGE *page, + WT_UPDATE *chain, bool commit, WT_UPDATE **fix_updp, bool *upd_appended, + bool first_committed_upd_in_hs) { WT_DECL_ITEM(hs_value); WT_DECL_RET; @@ -769,10 +771,10 @@ __txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_PAGE * *fix_updp = upd; /* - * When the prepared update is getting committed, use the retrieved history store update to form - * a proper stop timestamp in the history store. + * When the prepared update is getting committed or the history store update is still on the + * update chain, no need to append it onto the update chain. */ - if (commit) + if (commit || first_committed_upd_in_hs) goto done; /* @@ -1079,14 +1081,14 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_DECL_RET; WT_ITEM hs_recno_key; WT_TXN *txn; - WT_UPDATE *fix_upd, *tombstone, *upd; + WT_UPDATE *first_committed_upd, *fix_upd, *tombstone, *upd; #ifdef HAVE_DIAGNOSTIC WT_UPDATE *head_upd; #endif size_t not_used; uint8_t *p, hs_recno_key_buf[WT_INTPACK64_MAXSIZE]; char ts_string[3][WT_TS_INT_STRING_SIZE]; - bool upd_appended; + bool first_committed_upd_in_hs, prepare_on_disk, upd_appended; hs_cursor = NULL; txn = session->txn; @@ -1125,9 +1127,23 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, if (upd == NULL || upd->prepare_state != WT_PREPARE_INPROGRESS) goto prepare_verify; - WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); + /* A prepared operation that is rolled back will not have a timestamp worth asserting on. */ + if (commit) + WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); + + for (first_committed_upd = upd; first_committed_upd != NULL && + (first_committed_upd->txnid == WT_TXN_ABORTED || + first_committed_upd->prepare_state == WT_PREPARE_INPROGRESS); + first_committed_upd = first_committed_upd->next) + ; + /* - * Retrieve the previous update from the history store and append it to the update chain. + * Locate the previous update from the history store and append it to the update chain if + * required. We know there may be content in the history store if the prepared update is written + * to the disk image or first committed update older than the prepared update is marked as + * WT_UPDATE_HS. The second case is rare but can happen if the eviction that writes the prepared + * update to the disk image fails after it has inserted the other updates of the key into the + * history store. * * We need to do this before we resolve the prepared updates because if we abort the prepared * updates first, the history search logic may race with other sessions modifying the same key @@ -1137,10 +1153,13 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * the update is also from the same prepared transaction, restore the update from history store * or remove the key. */ - if (F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS) && + prepare_on_disk = F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS) && (upd->type != WT_UPDATE_TOMBSTONE || (!commit && upd->next != NULL && upd->durable_ts == upd->next->durable_ts && - upd->txnid == upd->next->txnid && upd->start_ts == upd->next->start_ts))) { + upd->txnid == upd->next->txnid && upd->start_ts == upd->next->start_ts)); + first_committed_upd_in_hs = + first_committed_upd != NULL && F_ISSET(first_committed_upd, WT_UPDATE_HS); + if (prepare_on_disk || first_committed_upd_in_hs) { btree = S2BT(session); cbt = (WT_CURSOR_BTREE *)(*cursorp); @@ -1160,6 +1179,9 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, hs_cursor->set_key(hs_cursor, 4, btree->id, &hs_recno_key, WT_TS_MAX, UINT64_MAX); } WT_ERR_NOTFOUND_OK(__wt_curhs_search_near_before(session, hs_cursor), true); + + /* We should only get not found if the prepared update is on disk. */ + WT_ASSERT(session, ret != WT_NOTFOUND || prepare_on_disk); if (ret == WT_NOTFOUND && !commit) { /* * Allocate a tombstone and prepend it to the row so when we reconcile the update chain @@ -1182,8 +1204,8 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_ERR(ret); tombstone = NULL; } else if (ret == 0) - WT_ERR(__txn_append_hs_record( - session, hs_cursor, cbt->ref->page, upd, commit, &fix_upd, &upd_appended)); + WT_ERR(__txn_locate_hs_record(session, hs_cursor, cbt->ref->page, upd, commit, &fix_upd, + &upd_appended, first_committed_upd)); else ret = 0; } @@ -1250,8 +1272,12 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * prepared updates are written to the data store. When the page is read back into memory, there * will be only one uncommitted prepared update. */ - if (fix_upd != NULL) + if (fix_upd != NULL) { WT_ERR(__txn_fixup_prepared_update(session, hs_cursor, fix_upd, commit)); + /* Clear the WT_UPDATE_HS flag as we should have removed it from the history store. */ + if (first_committed_upd_in_hs && !commit) + F_CLR(first_committed_upd, WT_UPDATE_HS); + } prepare_verify: #ifdef HAVE_DIAGNOSTIC diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 5bc0f245497..3ec6bb95934 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -639,7 +639,7 @@ err: */ static int __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, WT_ROW *rip, - wt_timestamp_t rollback_timestamp, uint64_t recno) + wt_timestamp_t rollback_timestamp, uint64_t recno, bool *is_ondisk_stable) { WT_CELL *kcell; WT_CELL_UNPACK_KV *vpack, _vpack; @@ -654,6 +654,10 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W vpack = &_vpack; upd = NULL; + /* Initialize the on-disk stable version flag. */ + if (is_ondisk_stable != NULL) + *is_ondisk_stable = false; + /* * Assert an exclusive or for rip and cip such that either only a cip for a column store or a * rip for a row store are passed into the function. @@ -768,9 +772,12 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W __wt_timestamp_to_string(vpack->tw.durable_stop_ts, ts_string[4]), vpack->tw.stop_txn, prepared ? "true" : "false"); } - } else + } else { /* Stable version according to the timestamp. */ + if (is_ondisk_stable != NULL) + *is_ondisk_stable = true; return (0); + } if (rip != NULL) WT_ERR(__rollback_row_modify(session, page, rip, upd)); @@ -821,9 +828,19 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r kcell = WT_COL_PTR(page, cip); __wt_cell_unpack_kv(session, page->dsk, kcell, &unpack); rle = __wt_cell_rle(&unpack); - for (j = 0; j < rle; j++) - WT_RET(__rollback_abort_ondisk_kv( - session, ref, cip, NULL, rollback_timestamp, recno + j)); + if (unpack.type != WT_CELL_DEL) { + for (j = 0; j < rle; j++) { + WT_RET(__rollback_abort_ondisk_kv(session, ref, cip, NULL, rollback_timestamp, + recno + j, &stable_update_found)); + /* Skip processing all RLE if the on-disk version is stable. */ + if (stable_update_found) { + if (rle > 1) + WT_STAT_CONN_DATA_INCR(session, txn_rts_stable_rle_skipped); + break; + } + } + } else + WT_STAT_CONN_DATA_INCR(session, txn_rts_delete_rle_skipped); recno += rle; } else { recno++; @@ -910,7 +927,8 @@ __rollback_abort_row_leaf(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t * If there is no stable update found in the update list, abort any on-disk value. */ if (!stable_update_found) - WT_ERR(__rollback_abort_ondisk_kv(session, ref, NULL, rip, rollback_timestamp, 0)); + WT_ERR( + __rollback_abort_ondisk_kv(session, ref, NULL, rip, rollback_timestamp, 0, NULL)); } /* Mark the page as dirty to reconcile the page. */ diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 9fc79a78d72..9faa37d0c57 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -714,7 +714,19 @@ __wt_txn_set_prepare_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t prepare_ WT_RET_MSG(session, EINVAL, "commit timestamp should not have been set before the prepare timestamp"); - WT_RET(__txn_assert_after_reads(session, "prepare", prepare_ts)); + /* + * Allows setting the prepared timestamp smaller than or equal to the latest active read + * timestamp. This feature is necessary to find the largest key in the table even if that + * key has been deleted. Set this flag cautiously as it breaks repeated reads. + */ +#ifdef HAVE_DIAGNOSTIC + if (!F_ISSET(txn, WT_TXN_TS_ROUND_PREPARED)) + WT_RET(__txn_assert_after_reads(session, "prepare", prepare_ts)); + else + WT_RET(__wt_msg(session, + "Skip checking prepare timestamp %s against the latest active read timestamp.", + __wt_timestamp_to_string(prepare_ts, ts_string[0]))); +#endif /* * Check whether the prepare timestamp is less than the oldest timestamp. diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index 565660c5b66..727cf132ffc 100755 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -13,12 +13,13 @@ #define STRING_MATCH_CONFIG(s, item) \ (strncmp(s, (item).str, (item).len) == 0 && (s)[(item).len] == '\0') -static int dump_config(WT_SESSION *, const char *, WT_CURSOR *, bool, bool); +static int dump_config(WT_SESSION *, const char *, WT_CURSOR *, bool, bool, bool); static int dump_json_begin(WT_SESSION *); static int dump_json_end(WT_SESSION *); static int dump_json_separator(WT_SESSION *); static int dump_json_table_end(WT_SESSION *); -static int dump_prefix(WT_SESSION *, bool, bool); +static const char *get_dump_type(bool, bool, bool); +static int dump_prefix(WT_SESSION *, bool, bool, bool); static int dump_record(WT_CURSOR *, bool, bool); static int dump_suffix(WT_SESSION *, bool); static int dump_table_config(WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, bool); @@ -33,12 +34,16 @@ usage(void) static const char *options[] = {"-c checkpoint", "dump as of the named checkpoint (the default is the most recent version of the data)", "-f output", "dump to the specified file (the default is stdout)", "-j", - "dump in JSON format", "-p", "dump in human readable format (pretty-print)", "-r", - "dump in reverse order", "-t timestamp", + "dump in JSON format", "-p", + "dump in human readable format (pretty-print). The -p flag can be combined with -x. In this " + "case, raw data elements will be formatted like -x with hexadecimal encoding.", + "-r", "dump in reverse order", "-t timestamp", "dump as of the specified timestamp (the default is the most recent version of the data)", "-x", "dump all characters in a hexadecimal encoding (by default printable characters are not " - "encoded)", + "encoded). The -x flag can be combined with -p. In this case, the dump will be formatted " + "similar to -p except for raw data elements, which will look like -x with hexadecimal " + "encoding.", NULL, NULL}; util_usage( @@ -108,8 +113,13 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) ++format_specifiers; if (hex) ++format_specifiers; - if (format_specifiers > 1) { - fprintf(stderr, "%s: the -j, -p and -x dump options are incompatible\n", progname); + + /* Supported options are -j, -p, -x and a combination of -p and -x. */ + if (format_specifiers > 1 && !(pretty && hex)) { + fprintf(stderr, + "%s: the only possible options are -j, -p, -x and a combination of -p and -x. Other " + "options are incompatible\n", + progname); return (usage()); } @@ -119,7 +129,7 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) else if ((fp = fopen(ofile, "w")) == NULL) return (util_err(session, errno, "%s: open", ofile)); - if (json && (dump_json_begin(session) != 0 || dump_prefix(session, hex, json) != 0)) + if (json && (dump_json_begin(session) != 0 || dump_prefix(session, pretty, hex, json) != 0)) goto err; WT_RET(__wt_scr_alloc(session_impl, 0, &tmp)); @@ -147,8 +157,7 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) WT_ERR(__wt_buf_set(session_impl, tmp, "", 0)); if (checkpoint != NULL) WT_ERR(__wt_buf_catfmt(session_impl, tmp, "checkpoint=%s,", checkpoint)); - WT_ERR(__wt_buf_catfmt(session_impl, tmp, "dump=%s", - json ? "json" : (hex ? "hex" : (pretty ? "pretty" : "print")))); + WT_ERR(__wt_buf_catfmt(session_impl, tmp, "dump=%s", get_dump_type(pretty, hex, json))); if ((ret = session->open_cursor(session, uri, NULL, (char *)tmp->data, &cursor)) != 0) { fprintf(stderr, "%s: cursor open(%s) failed: %s\n", progname, uri, session->strerror(session, ret)); @@ -172,7 +181,7 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) /* Set the "ignore tombstone" flag on the underlying cursor. */ F_SET(hs_dump_cursor->child, WT_CURSTD_IGNORE_TOMBSTONE); } - if (dump_config(session, simpleuri, cursor, hex, json) != 0) + if (dump_config(session, simpleuri, cursor, pretty, hex, json) != 0) goto err; if (dump_record(cursor, reverse, json) != 0) @@ -241,7 +250,8 @@ time_pair_to_timestamp(WT_SESSION_IMPL *session_impl, char *ts_string, WT_ITEM * * Dump the config for the uri. */ static int -dump_config(WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool hex, bool json) +dump_config( + WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool pretty, bool hex, bool json) { WT_CURSOR *mcursor; WT_DECL_RET; @@ -260,7 +270,7 @@ dump_config(WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool hex, b */ mcursor->set_key(mcursor, uri); if ((ret = mcursor->search(mcursor)) == 0) { - if ((!json && dump_prefix(session, hex, json) != 0) || + if ((!json && dump_prefix(session, pretty, hex, json) != 0) || dump_table_config(session, mcursor, cursor, uri, json) != 0 || dump_suffix(session, json) != 0) ret = 1; @@ -560,11 +570,35 @@ match: } /* + * Returns dump type string based on the passed format flags + */ +static const char * +get_dump_type(bool pretty, bool hex, bool json) +{ + const char *result; + + result = NULL; + + if (json) + result = "json"; + else if (hex && pretty) + result = "pretty_hex"; + else if (hex) + result = "hex"; + else if (pretty) + result = "pretty"; + else + result = "print"; + + return result; +} + +/* * dump_prefix -- * Output the dump file header prefix. */ static int -dump_prefix(WT_SESSION *session, bool hex, bool json) +dump_prefix(WT_SESSION *session, bool pretty, bool hex, bool json) { int vmajor, vminor, vpatch; @@ -577,7 +611,8 @@ dump_prefix(WT_SESSION *session, bool hex, bool json) if (!json && (fprintf(fp, "WiredTiger Dump (WiredTiger Version %d.%d.%d)\n", vmajor, vminor, vpatch) < 0 || - fprintf(fp, "Format=%s\n", hex ? "hex" : "print") < 0 || fprintf(fp, "Header\n") < 0)) + fprintf(fp, "Format=%s\n", (pretty && hex) ? "print hex" : hex ? "hex" : "print") < 0 || + fprintf(fp, "Header\n") < 0)) return (util_err(session, EIO, NULL)); return (0); diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 3bb80e94156..b7e7a9a952b 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -82,7 +82,6 @@ clock_thread(void *arg) testutil_check(g.conn->open_session(g.conn, NULL, NULL, &wt_session)); session = (WT_SESSION_IMPL *)wt_session; - g.ts_stable = 0; while (g.running) { __wt_writelock(session, &g.clock_lock); ++g.ts_stable; diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 83f1b1f6cef..962b1893305 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -5,22 +5,34 @@ set -e # Bypass this test for valgrind test "$TESTUTIL_BYPASS_VALGRIND" = "1" && exit 0 -# Temporarily disabled # Smoke-test checkpoints as part of running "make check". -#echo "checkpoint: 3 mixed tables" -#$TEST_WRAPPER ./t -T 3 -t m -# Temporarily disabled -#echo "checkpoint: 6 column-store tables" -#$TEST_WRAPPER ./t -T 6 -t c +echo "checkpoint: 3 mixed tables" +$TEST_WRAPPER ./t -T 3 -t m -# Temporarily disabled -#echo "checkpoint: 6 LSM tables" -#$TEST_WRAPPER ./t -T 6 -t l +echo "checkpoint: 6 column-store tables" +$TEST_WRAPPER ./t -T 6 -t c -# Temporarily disabled -#echo "checkpoint: 6 mixed tables" -#$TEST_WRAPPER ./t -T 6 -t m +echo "checkpoint: 6 column-store tables, named checkpoint" +$TEST_WRAPPER ./t -c 'TeSt' -T 6 -t c + +echo "checkpoint: 6 column-store tables with prepare" +$TEST_WRAPPER ./t -T 6 -t c -p + +echo "checkpoint: 6 column-store tables, named checkpoint with prepare" +$TEST_WRAPPER ./t -c 'TeSt' -T 6 -t c -p + +echo "checkpoint: column-store tables, stress history store. Sweep and timestamps" +$TEST_WRAPPER ./t -t c -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB + +echo "checkpoint: column-store tables, Sweep and timestamps" +$TEST_WRAPPER ./t -t c -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB + +echo "checkpoint: 6 LSM tables" +$TEST_WRAPPER ./t -T 6 -t l + +echo "checkpoint: 6 mixed tables" +$TEST_WRAPPER ./t -T 6 -t m echo "checkpoint: 6 row-store tables" $TEST_WRAPPER ./t -T 6 -t r @@ -34,18 +46,14 @@ $TEST_WRAPPER ./t -T 6 -t r -p echo "checkpoint: 6 row-store tables, named checkpoint with prepare" $TEST_WRAPPER ./t -c 'TeSt' -T 6 -t r -p -# Temporarily disabled -#echo "checkpoint: row-store tables, stress history store. Sweep and timestamps" -#$TEST_WRAPPER ./t -t r -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB +echo "checkpoint: row-store tables, stress history store. Sweep and timestamps" +$TEST_WRAPPER ./t -t r -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB -# Temporarily disabled -#echo "checkpoint: row-store tables, Sweep and timestamps" -#$TEST_WRAPPER ./t -t r -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB +echo "checkpoint: row-store tables, Sweep and timestamps" +$TEST_WRAPPER ./t -t r -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB -# Temporarily disabled -#echo "checkpoint: 3 mixed tables, with sweep" -#$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -s -n 100000 -k 100000 +echo "checkpoint: 3 mixed tables, with sweep" +$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -s -n 100000 -k 100000 -# Temporarily disabled -#echo "checkpoint: 3 mixed tables, with timestamps" -#$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -x -n 100000 -k 100000 +echo "checkpoint: 3 mixed tables, with timestamps" +$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -x -n 100000 -k 100000 diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index b236cb14308..90378cc0de9 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -140,6 +140,8 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(g.home, 512, working_dir); + g.ts_stable = 0; + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { cleanup(cnt == 1); /* Clean up previous runs */ @@ -171,7 +173,7 @@ main(int argc, char *argv[]) free(g.cookies); g.cookies = NULL; if ((ret = wt_shutdown()) != 0) { - (void)log_print_err("Start workers failed", ret, 1); + (void)log_print_err("Shutdown failed", ret, 1); break; } } @@ -250,7 +252,6 @@ cleanup(bool remove_dir) g.running = 0; g.ntables_created = 0; g.ts_oldest = 0; - g.ts_stable = 0; if (remove_dir) testutil_make_work_dir(g.home); diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index 2cd947fbd79..05b9a83b75b 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -280,11 +280,12 @@ real_worker(void) } else testutil_check(__wt_snprintf( buf, sizeof(buf), "commit_timestamp=%x", g.ts_stable + 1)); - __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); if ((ret = session->commit_transaction(session, buf)) != 0) { + __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); (void)log_print_err("real_worker:commit_transaction", ret, 1); goto err; } + __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); start_txn = true; /* Occasionally reopen cursors after committing. */ if (next_rnd % 13 == 0) { diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt index f5b5793491d..99707d70d08 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt @@ -26,9 +26,9 @@ runtime_monitor= enabled=true, limit=1900000000, ), - # Seems to insert around 490K records. Give it +-25K margin. + # Seems to insert around 477K records. Give it +-20K margin. # Seems to remove 180K records. Give it a similar margin. - postrun_statistics=[cache_hs_insert:470000:515000, cc_pages_removed:170000:200000] + postrun_statistics=[cache_hs_insert:457000:497000, cc_pages_removed:170000:200000] ), timestamp_manager= ( diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx index e49590da7d7..4ce0de2e381 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx @@ -137,8 +137,8 @@ test::run() /* The test will run for the duration as defined in the config. */ duration_seconds = _config->get_int(DURATION_SECONDS); testutil_assert(duration_seconds >= 0); - logger::log_msg( - LOG_INFO, "Waiting {" + std::to_string(duration_seconds) + "} for testing to complete."); + logger::log_msg(LOG_INFO, + "Waiting {" + std::to_string(duration_seconds) + "} seconds for testing to complete."); std::this_thread::sleep_for(std::chrono::seconds(duration_seconds)); /* End the test by calling finish on all known components. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx index 7cc113f8ad8..3b7d2d312aa 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx @@ -69,7 +69,7 @@ timestamp_manager::load() void timestamp_manager::do_work() { - std::string config; + std::string config, log_msg; /* latest_ts_s represents the time component of the latest timestamp provided. */ wt_timestamp_t latest_ts_s; @@ -82,7 +82,7 @@ timestamp_manager::do_work() */ testutil_assert(latest_ts_s >= _stable_ts); if ((latest_ts_s - _stable_ts) > _stable_lag) { - logger::log_msg(LOG_INFO, "Timestamp_manager: Stable timestamp expired."); + log_msg = "Timestamp_manager: Stable timestamp expired."; _stable_ts = latest_ts_s; config += std::string(STABLE_TS) + "=" + decimal_to_hex(_stable_ts); } @@ -94,13 +94,19 @@ timestamp_manager::do_work() wt_timestamp_t new_oldest_ts = _oldest_ts; testutil_assert(_stable_ts >= _oldest_ts); if ((_stable_ts - _oldest_ts) > _oldest_lag) { - logger::log_msg(LOG_INFO, "Timestamp_manager: Oldest timestamp expired."); + if (log_msg.empty()) + log_msg = "Timestamp_manager: Oldest timestamp expired."; + else + log_msg += " Oldest timestamp expired."; new_oldest_ts = _stable_ts - _oldest_lag; if (!config.empty()) config += ","; config += std::string(OLDEST_TS) + "=" + decimal_to_hex(new_oldest_ts); } + if (!log_msg.empty()) + logger::log_msg(LOG_INFO, log_msg); + /* * Save the new timestamps. Any timestamps that we're viewing from another thread should be set * AFTER we've saved the new timestamps to avoid races where we sweep data that is not yet diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx index e18125446fe..ebd73acf286 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx @@ -52,8 +52,9 @@ populate_worker(thread_context *tc) for (uint64_t i = 0; i < tc->key_count; ++i) { /* Start a txn. */ tc->transaction.begin(); - if (!tc->insert(cursor, coll.id, i)) { - /* We failed to insert, and our transaction was rolled back retry. */ + if (tc->insert(cursor, coll.id, i)) { + /* We failed to insert, rollback our transaction and retry. */ + tc->transaction.rollback(); --i; continue; } @@ -162,24 +163,31 @@ database_operation::insert_operation(thread_context *tc) auto &cc = ccv[counter]; while (tc->transaction.active() && tc->running()) { /* Insert a key value pair. */ - if (!tc->insert(cc.cursor, cc.coll.id, start_key + added_count)) { - committed = false; - break; + bool rollback_required = tc->insert(cc.cursor, cc.coll.id, start_key + added_count); + if (!rollback_required) { + added_count++; + if (tc->transaction.can_commit()) { + rollback_required = tc->transaction.commit(); + if (!rollback_required) + /* + * We need to inform the database model that we've added these keys as some + * other thread may rely on the key_count data. Only do so if we + * successfully committed. + */ + cc.coll.increase_key_count(added_count); + } } - added_count++; - tc->transaction.try_commit(); + + if (rollback_required) { + added_count = 0; + tc->transaction.rollback(); + } + /* Sleep the duration defined by the op_rate. */ tc->sleep(); } /* Reset our cursor to avoid pinning content. */ testutil_check(cc.cursor->reset(cc.cursor.get())); - - /* - * We need to inform the database model that we've added these keys as some other thread may - * rely on the key_count data. Only do so if we successfully committed. - */ - if (committed) - cc.coll.increase_key_count(added_count); counter++; if (counter == collections_per_thread) counter = 0; @@ -208,9 +216,17 @@ database_operation::read_operation(thread_context *tc) tc->transaction.begin(); while (tc->transaction.active() && tc->running()) { - if (tc->next(cursor) == WT_ROLLBACK) - /* We got an error, our transaction has been rolled back. */ - break; + auto ret = cursor->next(cursor.get()); + if (ret != 0) { + if (ret == WT_NOTFOUND) { + cursor->reset(cursor.get()); + } else if (ret == WT_ROLLBACK) { + tc->transaction.rollback(); + tc->sleep(); + continue; + } else + testutil_die(ret, "Unexpected error returned from cursor->next()"); + } tc->transaction.add_op(); tc->transaction.try_rollback(); tc->sleep(); @@ -263,21 +279,21 @@ database_operation::update_operation(thread_context *tc) /* Choose a random key to update. */ uint64_t key_id = random_generator::instance().generate_integer<uint64_t>(0, coll.get_key_count() - 1); - bool successful_update = tc->update(cursor, coll.id, tc->key_to_string(key_id)); + bool rollback_required = tc->update(cursor, coll.id, tc->key_to_string(key_id)); /* Reset our cursor to avoid pinning content. */ testutil_check(cursor->reset(cursor.get())); - /* We received a rollback in update. */ - if (!successful_update) - continue; - /* Commit the current transaction if we're able to. */ - tc->transaction.try_commit(); + if (!rollback_required && tc->transaction.can_commit()) + rollback_required = tc->transaction.commit(); + + if (rollback_required) + tc->transaction.rollback(); } - /* Make sure the last operation is committed now the work is finished. */ + /* Make sure the last operation is rolled back now the work is finished. */ if (tc->transaction.active()) - tc->transaction.commit(); + tc->transaction.rollback(); } } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx index 321cc45b61b..aadf992133e 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx @@ -71,6 +71,7 @@ transaction_context::begin(const std::string &config) random_generator::instance().generate_integer<int64_t>(_min_op_count, _max_op_count); _op_count = 0; _in_txn = true; + _needs_rollback = false; } void @@ -80,21 +81,22 @@ transaction_context::try_begin(const std::string &config) begin(config); } -void +/* It's possible to receive rollback in commit which is handled internally. */ +bool transaction_context::commit(const std::string &config) { + WT_DECL_RET; testutil_assert(_in_txn); - testutil_check( - _session->commit_transaction(_session, config.empty() ? nullptr : config.c_str())); - _op_count = 0; - _in_txn = false; -} - -void -transaction_context::try_commit(const std::string &config) -{ - if (can_commit_rollback()) - commit(config); + if ((ret = _session->commit_transaction(_session, config.empty() ? nullptr : config.c_str())) != + 0) { + logger::log_msg(LOG_WARN, + "Failed to commit transaction in commit, received error code: " + std::to_string(ret)); + _needs_rollback = true; + } else { + _op_count = 0; + _in_txn = false; + } + return (_needs_rollback); } void @@ -103,6 +105,7 @@ transaction_context::rollback(const std::string &config) testutil_assert(_in_txn); testutil_check( _session->rollback_transaction(_session, config.empty() ? nullptr : config.c_str())); + _needs_rollback = false; _op_count = 0; _in_txn = false; } @@ -110,7 +113,7 @@ transaction_context::rollback(const std::string &config) void transaction_context::try_rollback(const std::string &config) { - if (can_commit_rollback()) + if (can_rollback()) rollback(config); } @@ -124,8 +127,20 @@ transaction_context::set_commit_timestamp(wt_timestamp_t ts) testutil_check(_session->timestamp_transaction(_session, config.c_str())); } +void +transaction_context::set_needs_rollback(bool rollback) +{ + _needs_rollback = rollback; +} + bool -transaction_context::can_commit_rollback() +transaction_context::can_commit() +{ + return (!_needs_rollback && can_rollback()); +} + +bool +transaction_context::can_rollback() { return (_in_txn && _op_count >= _target_op_count); } @@ -184,8 +199,8 @@ thread_context::update(scoped_cursor &cursor, uint64_t collection_id, const std: ret = cursor->update(cursor.get()); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(); - return (false); + transaction.set_needs_rollback(true); + return (true); } else testutil_die(ret, "unhandled error while trying to update a key"); } @@ -193,14 +208,14 @@ thread_context::update(scoped_cursor &cursor, uint64_t collection_id, const std: tracking_operation::INSERT, collection_id, key.c_str(), value.c_str(), ts, op_track_cursor); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(); - return (false); + transaction.set_needs_rollback(true); + return (true); } else testutil_die( ret, "unhandled error while trying to save an update to the tracking table"); } transaction.add_op(); - return (true); + return (false); } bool @@ -226,8 +241,8 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k ret = cursor->insert(cursor.get()); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(); - return (false); + transaction.set_needs_rollback(true); + return (true); } else testutil_die(ret, "unhandled error while trying to insert a key"); } @@ -235,38 +250,14 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k tracking_operation::INSERT, collection_id, key.c_str(), value.c_str(), ts, op_track_cursor); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(); - return (false); + transaction.set_needs_rollback(true); + return (true); } else testutil_die( ret, "unhandled error while trying to save an insert to the tracking table"); } transaction.add_op(); - return (true); -} - -int -thread_context::next(scoped_cursor &cursor) -{ - WT_DECL_RET; - - ret = cursor->next(cursor.get()); - - if (ret == WT_NOTFOUND) { - testutil_check(cursor->reset(cursor.get())); - return (ret); - } - - if (ret == WT_ROLLBACK) { - transaction.rollback(); - testutil_check(cursor->reset(cursor.get())); - return (ret); - } - - if (ret != 0) - testutil_die(ret, "cursor->next() failed with an unexpected error."); - - return (0); + return (false); } void diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 5255740f59c..7bb6ac66cb6 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -71,17 +71,28 @@ class transaction_context { void begin(const std::string &config = ""); /* Begin a transaction if we are not currently in one. */ void try_begin(const std::string &config = ""); - void commit(const std::string &config = ""); - /* Attempt to commit the transaction given the requirements are met. */ - void try_commit(const std::string &config = ""); + /* + * Commit a transaction and return true if a rollback is required. + */ + bool commit(const std::string &config = ""); + /* Rollback a transaction, failure will abort the test. */ void rollback(const std::string &config = ""); /* Attempt to rollback the transaction given the requirements are met. */ void try_rollback(const std::string &config = ""); /* Set a commit timestamp. */ void set_commit_timestamp(wt_timestamp_t ts); - - private: - bool can_commit_rollback(); + /* Set that the transaction needs to be rolled back. */ + void set_needs_rollback(bool rollback); + /* + * Returns true if a transaction can be committed as determined by the op count and the state of + * the transaction. + */ + bool can_commit(); + /* + * Returns true if a transaction can be rolled back as determined by the op count and the state + * of the transaction. + */ + bool can_rollback(); private: /* @@ -99,6 +110,7 @@ class transaction_context { int64_t _max_op_count = INT64_MAX; int64_t _target_op_count = 0; bool _in_txn = false; + bool _needs_rollback = false; WT_SESSION *_session = nullptr; timestamp_manager *_timestamp_manager = nullptr; @@ -124,25 +136,17 @@ class thread_context { /* * Generic update function, takes a collection_id and key, will generate the value. * - * Returns true if it successfully updates the key, false if it receives rollback from the API. + * Returns true if a rollback is required. */ bool update(scoped_cursor &cursor, uint64_t collection_id, const std::string &key); /* * Generic insert function, takes a collection_id and key_id, will generate the value. * - * Returns true if it successfully inserts the key, false if it receives rollback from the API. + * Returns true if a rollback is required. */ bool insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t key_id); - /* - * Generic next function. - * - * Handles rollback and not found internally, but will return the error code to the caller so - * the caller can distinguish between them. - */ - int next(scoped_cursor &cursor); - void sleep(); bool running() const; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx index 4dd073cc115..03de6947a86 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx @@ -62,8 +62,24 @@ class hs_cleanup : public test { while (tc->running()) { tc->sleep(); - if (tc->next(cursor) != 0) - continue; + auto ret = cursor->next(cursor.get()); + if (ret != 0) { + if (ret == WT_NOTFOUND) { + cursor->reset(cursor.get()); + continue; + } + if (ret == WT_ROLLBACK) { + /* + * As a result of the logic in this test its possible that the previous next + * call can happen outside the context of a transaction. Assert that we are in + * one if we got a rollback. + */ + testutil_check(tc->transaction.can_rollback()); + tc->transaction.rollback(); + continue; + } + testutil_die(ret, "Unexpected error returned from cursor->next()"); + } testutil_check(cursor->get_key(cursor.get(), &key_tmp)); @@ -75,14 +91,17 @@ class hs_cleanup : public test { * API doesn't guarantee our buffer will still be valid once it is called, as such we * copy the buffer and then pass it into the API. */ - if (!tc->update(cursor, coll.id, key_value_t(key_tmp))) - continue; + bool rollback_required = tc->update(cursor, coll.id, key_value_t(key_tmp)); /* Commit our transaction. */ - tc->transaction.try_commit(); + if (!rollback_required && tc->transaction.can_commit()) + rollback_required = tc->transaction.commit(); + + if (rollback_required) + tc->transaction.rollback(); } /* Ensure our last transaction is resolved. */ if (tc->transaction.active()) - tc->transaction.commit(); + tc->transaction.rollback(); } }; diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index c863f47cbd3..26789a1ac9b 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -162,15 +162,9 @@ thread_run(void *arg) testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); -#if 0 - /* - * Make sure that alternative threads operate on column-store table - * - * FIXME-WT-6125: temporarily turn off column store test. - */ + /* Make alternate threads operate on the column-store table. */ if (td->id % 2 != 0) columnar_table = true; -#endif if (columnar_table) testutil_check(session->open_cursor(session, col_uri, NULL, NULL, &cursor)); @@ -384,7 +378,6 @@ recover_and_verify(uint32_t nthreads) fatal = false; for (i = 0; i < nthreads; ++i) { -#if 0 /* * Every alternative thread is operated on column-store table. Make sure that proper cursor * is used for verification of recovered records. @@ -396,11 +389,6 @@ recover_and_verify(uint32_t nthreads) columnar_table = false; cursor = row_cursor; } -#else - /* FIXME-WT-6125: temporarily turn off column store test. */ - columnar_table = false; - cursor = row_cursor; -#endif middle = 0; testutil_check(__wt_snprintf(fname[DELETE_RECORD_FILE_ID], diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index ec9fcec8175..6fa41f0d82c 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -150,11 +150,13 @@ thread_ts_run(void *arg) THREAD_DATA *td; int dbg; char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE]; + bool first; td = (THREAD_DATA *)arg; __wt_random_init(&rnd); testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + first = true; /* Update the oldest timestamp every 1 millisecond. */ for (;;) { /* @@ -166,13 +168,15 @@ thread_ts_run(void *arg) testutil_check(pthread_rwlock_unlock(&ts_lock)); testutil_assert(ret == 0 || ret == WT_NOTFOUND); if (ret == 0) { - /* - * Set both the oldest and stable timestamp so that we don't need to maintain read - * availability at older timestamps. - */ - testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), - "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string)); + /* Periodically let the oldest timestamp lag. */ + if (!first && __wt_random(&rnd) % 4 == 0) + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "stable_timestamp=%s", ts_string)); + else + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string)); testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + first = false; /* * Set and reset the checkpoint retention setting on a regular basis. We want to test * racing with the internal archive thread while we're here. diff --git a/src/third_party/wiredtiger/test/cursor_order/Makefile.am b/src/third_party/wiredtiger/test/cursor_order/Makefile.am index f5330b57d3b..448f8f95772 100644 --- a/src/third_party/wiredtiger/test/cursor_order/Makefile.am +++ b/src/third_party/wiredtiger/test/cursor_order/Makefile.am @@ -9,7 +9,7 @@ cursor_order_LDADD = $(top_builddir)/test/utility/libtest_util.la cursor_order_LDADD +=$(top_builddir)/libwiredtiger.la cursor_order_LDFLAGS = -static -TESTS = $(noinst_PROGRAMS) +TESTS = smoke.sh clean-local: rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/test/cursor_order/smoke.sh b/src/third_party/wiredtiger/test/cursor_order/smoke.sh new file mode 100755 index 00000000000..13165c82d3d --- /dev/null +++ b/src/third_party/wiredtiger/test/cursor_order/smoke.sh @@ -0,0 +1,14 @@ +#! /bin/sh + +set -e + +# Smoke-test cursors as part of running "make check". + +echo "cursor_order: rows" +$TEST_WRAPPER ./cursor_order -tr + +echo "cursor_order: variable-length columns" +$TEST_WRAPPER ./cursor_order -tv + +echo "cursor_order: fixed-length columns" +$TEST_WRAPPER ./cursor_order -tf diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 5222df53fc0..5bc5fa6580b 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -116,7 +116,7 @@ functions: mkdir -p cmake_build cd cmake_build $CMAKE \ - ${posix_configure_flags|-DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_STATIC=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL} -G "${cmake_generator|Ninja}" ./.. + ${posix_configure_flags|-DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_STATIC=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL} -G "${cmake_generator|Ninja}" ./.. fi elif [ "$OS" != "Windows_NT" ]; then # Compiling with Autoconf/Libtool. @@ -2623,7 +2623,20 @@ tasks: virtualenv -p python3 venv source venv/bin/activate pip3 install lorem pymongo - ./run_many_coll.sh ../../mongo/build/install/bin/mongod mongodb.log config/many-collection-testing many-collection clean-and-populate + ./run_many_coll.sh ../../mongo/build/opt/install/bin/mongod mongodb.log config/many-collection-testing many-collection clean-and-populate + - command: shell.exec + params: + working_dir: mongo-tests/largescale + shell: bash + silent: true + script: | + set -o errexit + set -o verbose + virtualenv -p python3 venv + source venv/bin/activate + pip3 install "pymongo[srv]" + res_dir=`find ./ -type d -name "many-collection-[0-9]*" -print` + ./upload-results-atlas.py ${atlas_wt_perf_test_user} ${atlas_wt_perf_pass} wt-perf-tests many-collection-test ${branch_name} $res_dir/results/results.json - name: cyclomatic-complexity commands: @@ -2689,7 +2702,7 @@ buildvariants: - ubuntu1804-test expansions: test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(echo "`grep -c ^processor /proc/cpuinfo` * 2" | bc) cmake_generator: Ninja @@ -3069,7 +3082,7 @@ buildvariants: batchtime: 10080 # 7 days expansions: test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) cmake_generator: Ninja @@ -3108,7 +3121,7 @@ buildvariants: batchtime: 10080 # 7 days expansions: test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) cmake_generator: Ninja diff --git a/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 b/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 index 57639d8b776..001a93dbab4 100644 --- a/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 +++ b/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 @@ -27,7 +27,7 @@ cd cmake_build # Configure build with CMake. if( $configure -eq $true) { - C:\cmake\bin\cmake --no-warn-unused-cli -DSWIG_DIR="C:\swigwin-3.0.2" -DSWIG_EXECUTABLE="C:\swigwin-3.0.2\swig.exe" -DCMAKE_BUILD_TYPE='None' -DENABLE_PYTHON=1 -DENABLE_STRICT=1 -DCMAKE_TOOLCHAIN_FILE='..\build_cmake\toolchains\cl.cmake' -G "Ninja" ..\. + C:\cmake\bin\cmake --no-warn-unused-cli -DSWIG_DIR="C:\swigwin-3.0.2" -DSWIG_EXECUTABLE="C:\swigwin-3.0.2\swig.exe" -DCMAKE_BUILD_TYPE='None' -DENABLE_PYTHON=1 -DENABLE_STRICT=1 -DCMAKE_TOOLCHAIN_FILE='..\cmake\toolchains\cl.cmake' -G "Ninja" ..\. } # Execute Ninja build. diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index a2fcc71c93e..461875e33bb 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -20,27 +20,27 @@ usage() { echo "usage: $0 [-aEFRSv] [-b format-binary] [-c config] [-e env-var]" echo " [-h home] [-j parallel-jobs] [-n total-jobs] [-r live-record-binary] [-t minutes] [format-configuration]" echo - echo " -a abort/recovery testing (defaults to off)" + echo " -a add configuration for abort/recovery testing (defaults to off)" echo " -b binary format binary (defaults to "./t")" echo " -c config format configuration file (defaults to CONFIG.stress)" - echo " -e envvar Environment variable setting (default to none)" echo " -E skip known errors (defaults to off)" + echo " -e envvar Environment variable setting (default to none)" echo " -F quit on first failure (defaults to off)" echo " -h home run directory (defaults to .)" echo " -j parallel jobs to execute in parallel (defaults to 8)" echo " -n total total jobs to execute (defaults to no limit)" - echo " -R run timing stress split test configurations (defaults to off)" + echo " -R add configuration for randomized split stress (defaults to none)" echo " -r binary record with UndoDB binary (defaults to no recording)" echo " -S run smoke-test configurations (defaults to off)" echo " -t minutes minutes to run (defaults to no limit)" echo " -v verbose output (defaults to off)" - echo " -- separates $name arguments from format arguments" + echo " -- separates $name arguments from additional format arguments" exit 1 } # Smoke-tests. -smoke_base_1="data_source=table rows=100000 threads=6 timer=4" +smoke_base_1="runs.source=table rows=100000 threads=6 timer=4" smoke_base_2="$smoke_base_1 leaf_page_max=9 internal_page_max=9" smoke_list=( # Three access methods. @@ -49,14 +49,14 @@ smoke_list=( # "$smoke_base_1 file_type=fix" # "$smoke_base_1 file_type=var" - # Huffman key/value encoding. + # Huffman value encoding. "$smoke_base_1 file_type=row huffman_value=1" # Temporarily disabled # "$smoke_base_1 file_type=var huffman_value=1" # LSM # Temporarily disabled - # "$smoke_base_1 file_type=row data_source=lsm" + # "$smoke_base_1 file_type=row runs.source=lsm" # Force the statistics server. "$smoke_base_1 file_type=row statistics_server=1" @@ -74,16 +74,16 @@ build="" config="CONFIG.stress" first_failure=0 format_args="" +format_binary="./t" home="." +live_record_binary="" minutes=0 parallel_jobs=8 -live_record_binary="" skip_errors=0 smoke_test=0 -timing_stress_split_test=0 +stress_split_test=0 total_jobs=0 verbose=0 -format_binary="./t" while :; do case "$1" in @@ -96,12 +96,12 @@ while :; do -c) config="$2" shift ; shift ;; - -e) - export "$2" - shift ; shift ;; -E) skip_errors=1 shift ;; + -e) + export "$2" + shift ; shift ;; -F) first_failure=1 shift ;; @@ -123,7 +123,7 @@ while :; do } shift ; shift ;; -R) - timing_stress_split_test=1 + stress_split_test=1 shift ;; -r) live_record_binary="$2" @@ -394,7 +394,7 @@ resolve() # Everything is a table unless explicitly a file. uri="table:wt" - grep 'data_source=file' $dir/CONFIG > /dev/null && uri="file:wt" + grep 'runs.source=file' $dir/CONFIG > /dev/null && uri="file:wt" # Use the wt utility to recover & verify the object. if $($wt_binary -m -R -h $dir verify $uri >> $log 2>&1); then @@ -474,25 +474,21 @@ format() dir="$home/RUNDIR.$count_jobs" log="$dir.log" + args="" if [[ $smoke_test -ne 0 ]]; then args=${smoke_list[$smoke_next]} smoke_next=$(($smoke_next + 1)) - echo "$name: starting smoke-test job in $dir ($(date))" - elif [[ $timing_stress_split_test -ne 0 ]]; then - args=$format_args - for k in {1..7}; do - args+=" timing_stress_split_$k=$(($RANDOM%2))" + fi + if [[ $abort_test -ne 0 ]]; then + args+=" format.abort=1" + fi + if [[ $stress_split_test -ne 0 ]]; then + for k in {1..8}; do + args+=" stress_split_$k=$(($RANDOM%2))" done - echo "$name: starting timing-stress-split job in $dir ($(date))" - else - args=$format_args - - # If abort/recovery testing is configured, do it 5% of the time. - [[ $abort_test -ne 0 ]] && - [[ $(($count_jobs % 20)) -eq 0 ]] && args="$args format.abort=1" - - echo "$name: starting job in $dir ($(date))" fi + args+=" $format_args" + echo "$name: starting job in $dir ($(date))" # If we're using UndoDB, append our default arguments. # diff --git a/src/third_party/wiredtiger/test/format/recover.sh b/src/third_party/wiredtiger/test/format/recover.sh deleted file mode 100644 index 6f65db73eb5..00000000000 --- a/src/third_party/wiredtiger/test/format/recover.sh +++ /dev/null @@ -1,52 +0,0 @@ -#! /bin/sh - -# Ulimit: don't drop core. -ulimit -c 0 - -# Timer: how many minutes format runs before aborting. -timer=2 - -# Runs: set to 0 to run infinitely. -runs=1000 -if test "$#" -gt "0"; then - runs=$1 -fi - -# Config: additional test/format configuration -config= - -# Assumes we're running in build_*/test/format directory. -tcmd=./t -wtcmd=../../wt -rundir2=RUNDIR.SAVE -count=0 -while true; do - count=`expr $count + 1` - if test $runs -eq 0; then - echo "recovery test: $count" - else - if test $count -gt $runs; then - exit 0 - fi - echo "recovery test: $count of $runs" - fi - - rm -rf $rundir2 - $tcmd $config -q format.abort=1 logging=1 runs.timer=$timer - - # Save a copy of the database directory exactly as it was at the crash. - cp -rp RUNDIR $rundir2 - - # - # Everything is a table unless explicitly a file. - # - isfile=`grep data_source RUNDIR/CONFIG | grep -c file || exit 0` - if test "$isfile" -ne 0; then - uri="file:wt" - else - uri="table:wt" - fi - - # We know we aborted, so force recovery to run. - $wtcmd -m -R -h RUNDIR verify $uri || exit 1 -done diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 3ea620e638e..ea8cb43ac1c 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -204,7 +204,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) + self.session.create(uri, 'key_format={},value_format=S,verbose=(write_timestamp),write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) @@ -438,5 +438,14 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c.close() ''' + # Confirm that rolling back after preparing doesn't fire an assertion. + c = self.session.open_cursor(uri) + self.session.begin_transaction() + c[key_ts6] = 'value24' + self.session.prepare_transaction( + 'prepare_timestamp=' + self.timestamp_str(24)) + self.session.rollback_transaction() + c.close() + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug022.py b/src/third_party/wiredtiger/test/suite/test_bug022.py index 31fb2387662..8e7b355133e 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug022.py +++ b/src/third_party/wiredtiger/test/suite/test_bug022.py @@ -30,27 +30,37 @@ # Testing that we don't allow modifies on top of tombstone updates. import wiredtiger, wttest +from wtscenario import make_scenarios class test_bug022(wttest.WiredTigerTestCase): uri = 'file:test_bug022' conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def get_key(self, i): + return str(i) if self.usestrings else i + def test_apply_modifies_on_onpage_tombstone(self): - self.session.create(self.uri, 'key_format=S,value_format=S') + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) value = 'a' * 500 for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value + cursor[self.get_key(i)] = value self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply tombstones for every key. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) cursor.remove() self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) @@ -59,11 +69,11 @@ class test_bug022(wttest.WiredTigerTestCase): # Now try to apply a modify on top of the tombstone at timestamp 3. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 0, 100)]), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() # Check that the tombstone is visible. for i in range(1, 10000): - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_dump01.py b/src/third_party/wiredtiger/test/suite/test_dump01.py new file mode 100644 index 00000000000..cd6de8c7f62 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_dump01.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# wt_util +# [END_TAGS] + +import os, shutil +import wiredtiger, wttest + +from suite_subprocess import suite_subprocess + +# test_pretty_hex_dump +# Utilities: wt dump +# Test the dump utility with pretty hex flag +class test_pretty_hex_dump(wttest.WiredTigerTestCase, suite_subprocess): + table_format = 'key_format=i,value_format=u' + uri = 'table:test_dump' + + pretty_dump_file = 'pretty_dump.out' + pretty_hex_dump_file = 'pretty_hex_dump.out' + hex_dump_file = 'hex.out' + + pretty_hex_format = 'Format=print hex\n' + data_header = 'Data\n' + + def get_bytes(self, i, len): + """ + Return a pseudo-random, but predictable string that uses all characters + """ + ret = b'' + for j in range(0, len // 3): + k = i + j + ret += bytes([k%255 + 1, (k*3)%255 + 1, (k*7)%255 + 1]) + return ret + bytes([0]) # Add a final null byte + + def populate_table(self, uri): + """ + Populate test_dump table + """ + cursor = self.session.open_cursor(uri, None, None) + for i in range(2, 15): + val_bytes = self.get_bytes(i*i, i*i) + cursor[i] = (val_bytes) + cursor.close() + + def test_dump(self): + """ + Generates test table with byte array values. After that dumps the table in hex, pretty and + pretty hex formats. And then valudates pretty hex dump. + """ + # Create three dumps: hex, pretty and pretty_hex + self.session.create(self.uri, self.table_format) + self.populate_table(self.uri) + + self.runWt(['dump', '-x', self.uri], outfilename=self.hex_dump_file) + self.runWt(['dump', '-p', self.uri], outfilename=self.pretty_dump_file) + self.runWt(['dump', '-px', self.uri], outfilename=self.pretty_hex_dump_file) + + # Validate pretty hex + hex = open(self.hex_dump_file).readlines() + pretty = open(self.pretty_dump_file).readlines() + pretty_hex = open(self.pretty_hex_dump_file).readlines() + + # First validate number of lines the dumps + self.assertEqual(True, len(pretty) == len(pretty_hex), + 'Pretty and pretty_hex output must have the same numbler of lines.') + self.assertEqual(True, len(hex) == len(pretty_hex), + 'Hex and pretty_hex output must have the same numbler of lines.') + + # Next analyse the pretty hex dump line by line + data_started = False + value_line = False + for h, p, px in zip(hex, pretty, pretty_hex): + if data_started: + # Data section started + if value_line: + # Test values + self.assertEqual(True, h == px, + 'Hex and pretty_hex values must match!\n' + 'Hex: ' + h + 'Pretty_hex: ' + px) + else: + # Test keys + self.assertEqual(True, p == px, + 'Pretty and pretty_hex keys must match!\n' + 'Pretty: ' + p + 'Pretty_hex: ' + px) + + value_line = not value_line + else: + if p.startswith('Format='): + # The two dumps must have appropriate format string + self.assertEqual(True, px == self.pretty_hex_format, + 'Format for pretty_hex dump is: ' + px + ' Expected: ' + self.pretty_hex_format) + else: + # The two dump lines must match + self.assertEqual(True, p == px, + 'Dump lines differ!\n' + 'Pretty: ' + p + ', Pretty_hex: ' + px) + + # Test if data sectio has started + if not data_started and p == self.data_header: + self.assertEqual(True, p == px, + 'Data section starts at different lines.\n' + 'Pretty: ' + p + 'Pretty_hex: ' + px) + self.assertEqual(True, h == px, + 'Data section starts at different lines.\n' + 'Hex: ' + h + 'Pretty_hex: ' + px) + data_started = True + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index deea2f3dfc5..ea3fe771169 100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -28,6 +28,7 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest +from wtscenario import make_scenarios # test_durable_ts03.py # Check that the checkpoint honors the durable timestamp of updates. @@ -35,11 +36,17 @@ class test_durable_ts03(wttest.WiredTigerTestCase): conn_config = 'cache_size=10MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_durable_ts03(self): # Create a table. uri = 'table:test_durable_ts03' nrows = 3000 - self.session.create(uri, 'key_format=i,value_format=u') + self.session.create(uri, 'key_format={},value_format=u'.format(self.key_format)) valueA = b"aaaaa" * 100 valueB = b"bbbbb" * 100 valueC = b"ccccc" * 100 @@ -51,7 +58,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): # Load the data into the table. session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueA session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) @@ -65,7 +72,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): # Update all the values within transaction. Commit the transaction with # a durable timestamp newer than the stable timestamp. cursor = session.open_cursor(uri, None) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueB session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) @@ -105,7 +112,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEqual(value, valueA) self.assertEquals(cursor.reset(), 0) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueC session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(220)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py index b4e65f183b0..de9a7475817 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare06.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -90,24 +90,18 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): s_reader = self.conn.open_session() s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(40)) - # It is illegal to set the prepare timestamp as earlier than an active - # read timestamp even with roundup_timestamps settings. This is only + # It is legal to set the prepare timestamp earlier than an active + # read timestamp with roundup_timestamps settings. This is only # checked in diagnostic builds. if wiredtiger.diagnostic_build(): self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(10)), - "/must be greater than the latest active read timestamp/") + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)), 0) self.session.rollback_transaction() - # It is illegal to set the prepare timestamp the same as an active read - # timestamp even with roundup_timestamps settings. + # It is legal to set the prepare timestamp the same as an active read + # timestamp with roundup_timestamps settings. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(40)), - "/must be greater than the latest active read timestamp/") + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)), 0) self.session.rollback_transaction() ''' @@ -144,5 +138,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): s_reader.commit_transaction() + self.ignoreStdoutPatternIfExists("Skip checking prepare timestamp") + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index 5593c75f3ff..b12bf3d12a4 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -30,7 +30,10 @@ import os, wiredtiger, wttest FileSystem = wiredtiger.FileSystem # easy access to constants # test_tiered06.py -# Test the local storage source. +# Test the local storage source's file system implementation. +# Note that the APIs we are testing are not meant to be used directly +# by any WiredTiger application, these APIs are used internally. +# However, it is useful to do tests of this API independently. class test_tiered06(wttest.WiredTigerTestCase): # Load the local store extension. def conn_extensions(self, extlist): @@ -64,16 +67,26 @@ class test_tiered06(wttest.WiredTigerTestCase): # The object doesn't exist yet. self.assertFalse(fs.fs_exist(session, 'foobar')) - fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_create) + # We cannot use the file system to create files, it is readonly. + # So use python I/O to build up the file. + f = open('foobar', 'wb') - # Just like a regular file system, the object exists now. - self.assertTrue(fs.fs_exist(session, 'foobar')) + # The object still doesn't exist yet. + self.assertFalse(fs.fs_exist(session, 'foobar')) outbytes = ('MORE THAN ENOUGH DATA\n'*100000).encode() - fh.fh_write(session, 0, outbytes) + f.write(outbytes) + f.close() - # The object exists after close - fh.close(session) + # Nothing is in the directory list until a flush. + self.assertEquals(fs.fs_directory_list(session, '', ''), []) + + # Flushing moves the file into the file system + local.ss_flush(session, fs, 'foobar', 'foobar', None) + local.ss_flush_finish(session, fs, 'foobar', 'foobar', None) + + # The object exists now. + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) self.assertTrue(fs.fs_exist(session, 'foobar')) fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) @@ -90,39 +103,14 @@ class test_tiered06(wttest.WiredTigerTestCase): fh.fh_lock(session, False) fh.close(session) - # Nothing is in the directory list until a flush. - self.assertEquals(fs.fs_directory_list(session, '', ''), []) - - fh = fs.fs_open_file(session, 'zzz', FileSystem.open_file_type_data, FileSystem.open_create) - - # Sync merely syncs to the local disk. - fh.fh_sync(session) - fh.close(session) # zero length - self.assertEquals(sorted(fs.fs_directory_list(session, '', '')), []) - - # See that we can rename objects. - fs.fs_rename(session, 'zzz', 'yyy', 0) - self.assertEquals(sorted(fs.fs_directory_list(session, '', '')), []) - - # See that we can remove objects. - fs.fs_remove(session, 'yyy', 0) - - # Nothing is in the directory list until a flush. - self.assertEquals(fs.fs_directory_list(session, '', ''), []) - - # Flushing moves the file. - local.ss_flush(session, fs, 'foobar', 'foobar', None) - local.ss_flush_finish(session, fs, 'foobar', 'foobar', None) - self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) - # Files that have been flushed cannot be manipulated. - with self.expectedStderrPattern('foobar: rename of flushed file not allowed'): + with self.expectedStderrPattern('foobar: rename of file not supported'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: fs.fs_rename(session, 'foobar', 'barfoo', 0)) self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) # Files that have been flushed cannot be manipulated through the custom file system. - with self.expectedStderrPattern('foobar: remove of flushed file not allowed'): + with self.expectedStderrPattern('foobar: remove of file not supported'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: fs.fs_remove(session, 'foobar', 0)) self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) @@ -136,14 +124,17 @@ class test_tiered06(wttest.WiredTigerTestCase): session = self.session local = self.get_local_storage_source() + cachedir = ("objects_cache") os.mkdir("objects") - fs = local.ss_customize_file_system(session, "./objects", "Secret", None) + os.mkdir("objects_cache") + fs = local.ss_customize_file_system( + session, "./objects", "Secret", "cache_directory=" + cachedir) # We call these 4K chunks of data "blocks" for this test, but that doesn't # necessarily relate to WT block sizing. nblocks = 1000 block_size = 4096 - fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_create) + f = open('abc', 'wb') # blocks filled with 'a', etc. a_block = ('a' * block_size).encode() @@ -153,47 +144,63 @@ class test_tiered06(wttest.WiredTigerTestCase): # write all blocks as 'a', but in reverse order for pos in range(file_size - block_size, 0, -block_size): - fh.fh_write(session, pos, a_block) + f.seek(pos) + f.write(a_block) # write the even blocks as 'b', forwards for pos in range(0, file_size, block_size * 2): - fh.fh_write(session, pos, b_block) + f.seek(pos) + f.write(b_block) # write every third block as 'c', backwards for pos in range(file_size - block_size, 0, -block_size * 3): - fh.fh_write(session, pos, c_block) - fh.close(session) + f.seek(pos) + f.write(c_block) + f.close() + + # Flushing moves the file into the file system + local.ss_flush(session, fs, 'abc', 'abc', None) + local.ss_flush_finish(session, fs, 'abc', 'abc', None) + + # Use the file system to open and read the file. + # We do this twice, and between iterations, we remove the cached file to make sure + # it is copied back from the bucket directory. + # + # XXX: this uses knowledge of the implementation, but at the current time, + # we don't have a way via the API to "age out" a file from the cache. + for i in range(0, 2): + in_block = bytes(block_size) + fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_readonly) + + # Do some spot checks, reading non-sequentially + fh.fh_read(session, 500 * block_size, in_block) # divisible by 2, not 3 + self.assertEquals(in_block, b_block) + fh.fh_read(session, 333 * block_size, in_block) # divisible by 3, not 2 + self.assertEquals(in_block, c_block) + fh.fh_read(session, 401 * block_size, in_block) # not divisible by 2 or 3 + self.assertEquals(in_block, a_block) + + # Read the whole file, backwards checking to make sure + # each block was written correctly. + for block_num in range(nblocks - 1, 0, -1): + pos = block_num * block_size + fh.fh_read(session, pos, in_block) + if block_num % 3 == 0: + self.assertEquals(in_block, c_block) + elif block_num % 2 == 0: + self.assertEquals(in_block, b_block) + else: + self.assertEquals(in_block, a_block) + fh.close(session) + os.remove(os.path.join(cachedir, 'abc')) - in_block = bytes(block_size) - fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_readonly) - - # Do some spot checks, reading non-sequentially - fh.fh_read(session, 500 * block_size, in_block) # divisible by 2, not 3 - self.assertEquals(in_block, b_block) - fh.fh_read(session, 333 * block_size, in_block) # divisible by 3, not 2 - self.assertEquals(in_block, c_block) - fh.fh_read(session, 401 * block_size, in_block) # not divisible by 2 or 3 - self.assertEquals(in_block, a_block) - - # Read the whole file, backwards checking to make sure - # each block was written correctly. - for block_num in range(nblocks - 1, 0, -1): - pos = block_num * block_size - fh.fh_read(session, pos, in_block) - if block_num % 3 == 0: - self.assertEquals(in_block, c_block) - elif block_num % 2 == 0: - self.assertEquals(in_block, b_block) - else: - self.assertEquals(in_block, a_block) - fh.close(session) local.terminate(session) def create_with_fs(self, fs, fname): session = self.session - fh = fs.fs_open_file(session, fname, FileSystem.open_file_type_data, FileSystem.open_create) - fh.fh_write(session, 0, 'some stuff'.encode()) - fh.close(session) + f = open(fname, 'wb') + f.write('some stuff'.encode()) + f.close() objectdir1 = "./objects1" objectdir2 = "./objects2" @@ -298,7 +305,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # A flush copies to the cloud, nothing is removed. local.ss_flush(session, fs1, 'beagle.wt', 'beagle.wtobj') self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) - self.check_dirlist(fs1, '', []) + self.check_dirlist(fs1, '', ['beagle']) self.check_dirlist(fs2, '', []) self.check_caches([], []) self.check_objects(['beagle'], []) @@ -311,7 +318,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # It's okay to flush again, nothing changes local.ss_flush(session, fs1, 'beagle.wt', 'beagle.wtobj') self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) - self.check_dirlist(fs1, '', []) + self.check_dirlist(fs1, '', ['beagle']) self.check_dirlist(fs2, '', []) self.check_caches([], []) self.check_objects(['beagle'], []) @@ -333,13 +340,13 @@ class test_tiered06(wttest.WiredTigerTestCase): local.ss_flush_finish(session, fs1, 'bat.wt', 'bat.wtobj') self.check_home(['bird', 'bison', 'cougar', 'coyote', 'cub']) - self.check_dirlist(fs1, '', ['beagle', 'bat']) - self.check_dirlist(fs2, '', ['cat']) + self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison']) + self.check_dirlist(fs2, '', ['cat', 'cub']) self.check_caches(['beagle', 'bat'], ['cat']) self.check_objects(['beagle', 'bat', 'bison'], ['cat', 'cub']) # Test directory listing prefixes - self.check_dirlist(fs1, '', ['beagle', 'bat']) + self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison']) self.check_dirlist(fs1, 'ba', ['bat']) self.check_dirlist(fs1, 'be', ['beagle']) self.check_dirlist(fs1, 'x', []) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index 4af2004c162..be6dafc5c38 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -45,6 +45,10 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): nentries = 10 table_cnt = 3 + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] types = [ ('all', dict(use_stable='false', run_wt=0)), ('all+wt', dict(use_stable='false', run_wt=1)), @@ -56,7 +60,7 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): ('stable+wt', dict(use_stable='true', run_wt=1)), ('stable+wt2', dict(use_stable='true', run_wt=2)), ] - scenarios = make_scenarios(types) + scenarios = make_scenarios(key_format_values, types) def data_and_checkpoint(self): # @@ -64,10 +68,11 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): # Add data to each of them separately and checkpoint so that each one # has a different stable timestamp. # - self.session.create(self.oplog_uri, 'key_format=i,value_format=i') - self.session.create(self.coll1_uri, 'key_format=i,value_format=i,log=(enabled=false)') - self.session.create(self.coll2_uri, 'key_format=i,value_format=i,log=(enabled=false)') - self.session.create(self.coll3_uri, 'key_format=i,value_format=i,log=(enabled=false)') + basecfg = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.oplog_uri, basecfg) + self.session.create(self.coll1_uri, basecfg + ',log=(enabled=false)') + self.session.create(self.coll2_uri, basecfg + ',log=(enabled=false)') + self.session.create(self.coll3_uri, basecfg + ',log=(enabled=false)') c_op = self.session.open_cursor(self.oplog_uri) c = [] c.append(self.session.open_cursor(self.coll1_uri)) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp11.py b/src/third_party/wiredtiger/test/suite/test_timestamp11.py index c4b4210df43..d5998bdc212 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp11.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp11.py @@ -32,14 +32,24 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + def test_timestamp_range(self): base = 'timestamp11' uri = 'file:' + base - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + + key = 'key' if self.usestrings else 1 + key2 = 'key2' if self.usestrings else 2 # Test that mixed timestamp usage where some transactions use timestamps # and others don't behave in the expected way. @@ -49,8 +59,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(2)) - c['key'] = 'value2' - c['key2'] = 'value2' + c[key] = 'value2' + c[key2] = 'value2' self.session.commit_transaction() c.close() @@ -62,13 +72,13 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(5)) - c['key'] = 'value5' + c[key] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key2'] = 'valueNOTS' + c[key2] = 'valueNOTS' self.session.commit_transaction() c.close() @@ -85,15 +95,15 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - self.assertEquals(c['key'], 'value2') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'value2') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + stable_ts) - self.assertEquals(c['key'], 'value2') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'value2') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() @@ -104,13 +114,13 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(5)) - c['key2'] = 'value5' + c[key2] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key'] = 'valueNOTS' + c[key] = 'valueNOTS' self.session.commit_transaction() c.close() @@ -119,8 +129,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # Without a timestamp. We should see the latest value for each. c = self.session.open_cursor(uri) self.session.begin_transaction() - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'value5') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'value5') self.session.commit_transaction() c.close() @@ -128,8 +138,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # value at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + stable_ts) - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() @@ -138,8 +148,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # we inserted at timestamp 5 after the non-timestamped insert. c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'value5') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'value5') self.session.commit_transaction() c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp12.py b/src/third_party/wiredtiger/test/suite/test_timestamp12.py index e54ed923474..5ccab50ce05 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp12.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp12.py @@ -38,12 +38,16 @@ class test_timestamp12(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' coll_uri = 'table:collection12' oplog_uri = 'table:oplog12' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] closecfg = [ ('dfl', dict(close_cfg='', all_expected=False)), ('use_stable', dict(close_cfg='use_timestamp=true', all_expected=False)), ('all_dirty', dict(close_cfg='use_timestamp=false', all_expected=True)), - ] - scenarios = make_scenarios(closecfg) + ] + scenarios = make_scenarios(key_format_values, closecfg) def verify_expected(self, op_exp, coll_exp): c_op = self.session.open_cursor(self.oplog_uri) @@ -67,8 +71,9 @@ class test_timestamp12(wttest.WiredTigerTestCase): # Add data to each of them separately and checkpoint so that each one # has a different stable timestamp. # - self.session.create(self.oplog_uri, 'key_format=i,value_format=i') - self.session.create(self.coll_uri, 'key_format=i,value_format=i,log=(enabled=false)') + basecfg = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.oplog_uri, basecfg) + self.session.create(self.coll_uri, basecfg + ',log=(enabled=false)') c_op = self.session.open_cursor(self.oplog_uri) c_coll = self.session.open_cursor(self.coll_uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py index 8d6e753bf4d..3b4c4bf40ae 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp14.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -40,6 +40,12 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_all_durable_old(self): # This test was originally for testing the all_committed timestamp. # In the absence of prepared transactions, all_durable is identical to @@ -47,8 +53,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): all_durable_uri = self.uri + '_all_durable' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(all_durable_uri, 'key_format=i,value_format=i') - session2.create(all_durable_uri, 'key_format=i,value_format=i') + session1.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) # Scenario 0: No commit timestamp has ever been specified therefore # There is no all_durable timestamp and we will get an error @@ -132,8 +138,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): oldest_reader_uri = self.uri + '_oldest_reader_pinned' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(oldest_reader_uri, 'key_format=i,value_format=i') - session2.create(oldest_reader_uri, 'key_format=i,value_format=i') + session1.create(oldest_reader_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(oldest_reader_uri, 'key_format={},value_format=i'.format(self.key_format)) # Nothing is reading so there is no oldest reader. self.assertRaisesException(wiredtiger.WiredTigerError, @@ -190,7 +196,7 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): def test_pinned_oldest(self): pinned_oldest_uri = self.uri + 'pinned_oldest' session1 = self.setUpSessionOpen(self.conn) - session1.create(pinned_oldest_uri, 'key_format=i,value_format=i') + session1.create(pinned_oldest_uri, 'key_format={},value_format=i'.format(self.key_format)) # Confirm no oldest timestamp exists. self.assertRaisesException(wiredtiger.WiredTigerError, lambda: self.conn.query_timestamp('get=oldest')) @@ -241,7 +247,7 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): def test_all_durable(self): all_durable_uri = self.uri + '_all_durable' session1 = self.setUpSessionOpen(self.conn) - session1.create(all_durable_uri, 'key_format=i,value_format=i') + session1.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) # Since this is a non-prepared transaction, we'll be using the commit # timestamp when calculating all_durable since it's implied that they're @@ -329,8 +335,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): all_uri = self.uri + 'pinned_oldest' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(all_uri, 'key_format=i,value_format=i') - session2.create(all_uri, 'key_format=i,value_format=i') + session1.create(all_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(all_uri, 'key_format={},value_format=i'.format(self.key_format)) cur1 = session1.open_cursor(all_uri) cur2 = session2.open_cursor(all_uri) # Set up oldest timestamp. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp16.py b/src/third_party/wiredtiger/test/suite/test_timestamp16.py index f74557ad2c6..2485a14be12 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp16.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp16.py @@ -34,7 +34,6 @@ import random from suite_subprocess import suite_subprocess import wiredtiger, wttest -from wtscenario import make_scenarios class test_timestamp16(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp16' diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp17.py b/src/third_party/wiredtiger/test/suite/test_timestamp17.py index 3c2453b0aa2..fe4ec606d23 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp17.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp17.py @@ -43,8 +43,14 @@ class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_inconsistent_timestamping(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) self.session.begin_transaction() cur1 = self.session.open_cursor(self.uri) cur1[1] = 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp18.py b/src/third_party/wiredtiger/test/suite/test_timestamp18.py index c7f20dbd9a4..b41fc5ccb34 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp18.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp18.py @@ -41,15 +41,23 @@ from wtscenario import make_scenarios class test_timestamp18(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] non_ts_writes = [ ('insert', dict(delete=False)), ('delete', dict(delete=True)), ] - scenarios = make_scenarios(non_ts_writes) + scenarios = make_scenarios(key_format_values, non_ts_writes) + + def get_key(self, i): + return str(i) if self.usestrings else i def test_ts_writes_with_non_ts_write(self): uri = 'table:test_timestamp18' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -61,17 +69,17 @@ class test_timestamp18(wttest.WiredTigerTestCase): # A series of timestamped writes on each key. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Add a non-timestamped delete. @@ -80,10 +88,10 @@ class test_timestamp18(wttest.WiredTigerTestCase): for i in range(1, 10000): if i % 2 == 0: if self.delete: - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) cursor.remove() else: - cursor[str(i)] = value4 + cursor[self.get_key(i)] = value4 self.session.checkpoint() @@ -94,16 +102,16 @@ class test_timestamp18(wttest.WiredTigerTestCase): # invisible. if i % 2 == 0: if self.delete: - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: - self.assertEqual(cursor[str(i)], value4) + self.assertEqual(cursor[self.get_key(i)], value4) # Otherwise, expect one of the timestamped writes. else: if ts == 2: - self.assertEqual(cursor[str(i)], value1) + self.assertEqual(cursor[self.get_key(i)], value1) elif ts == 3: - self.assertEqual(cursor[str(i)], value2) + self.assertEqual(cursor[self.get_key(i)], value2) else: - self.assertEqual(cursor[str(i)], value3) + self.assertEqual(cursor[self.get_key(i)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py index 9f5b71df843..e271c8f1145 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp19.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -30,15 +30,22 @@ # Use the oldest timestamp in the metadata as the oldest timestamp on restart. import wiredtiger, wttest from wtdataset import SimpleDataSet +from wtscenario import make_scenarios class test_timestamp19(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def updates(self, uri, value, ds, nrows, commit_ts): session = self.session cursor = session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) @@ -46,11 +53,11 @@ class test_timestamp19(wttest.WiredTigerTestCase): def test_timestamp(self): uri = "table:test_timestamp19" - create_params = 'value_format=S,key_format=i' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_params) ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() nrows = 1000 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp20.py b/src/third_party/wiredtiger/test/suite/test_timestamp20.py index c4edcbfe592..7bfeb670004 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp20.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp20.py @@ -27,6 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_timestamp20.py # Exercise fixing up of out-of-order updates in the history store. @@ -34,9 +35,18 @@ class test_timestamp20(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def get_key(self, i): + return str(i) if self.usestrings else i + def test_timestamp20_standard(self): uri = 'table:test_timestamp20' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -48,17 +58,17 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) old_reader_session = self.conn.open_session() @@ -69,19 +79,19 @@ class test_timestamp20(wttest.WiredTigerTestCase): # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value4 + cursor[self.get_key(i)] = value4 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() - cursor[str(i)] = value5 + cursor[self.get_key(i)] = value5 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): - self.assertEqual(cursor[str(i)], value4) + self.assertEqual(cursor[self.get_key(i)], value4) self.session.rollback_transaction() for i in range(1, 10000): - self.assertEqual(old_reader_cursor[str(i)], value2) + self.assertEqual(old_reader_cursor[self.get_key(i)], value2) old_reader_session.rollback_transaction() # In this test we're using modifies since they are more sensitive to corruptions. @@ -90,7 +100,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # the conversion to a Python string. def test_timestamp20_modify(self): uri = 'table:test_timestamp20' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -101,19 +111,19 @@ class test_timestamp20(wttest.WiredTigerTestCase): # Apply the base value. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Now apply a series of modifies. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) @@ -129,7 +139,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # This will be the end of the chain of modifies. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) @@ -137,17 +147,17 @@ class test_timestamp20(wttest.WiredTigerTestCase): # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) # Open up a new transaction and read at 30. # We shouldn't be able to see past 5 due to txnid visibility. self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): - self.assertEqual(cursor[str(i)], value2) + self.assertEqual(cursor[self.get_key(i)], value2) self.session.rollback_transaction() # Put together expected value. @@ -157,5 +167,5 @@ class test_timestamp20(wttest.WiredTigerTestCase): # On the other hand, this older transaction SHOULD be able to read past the 5. for i in range(1, 10000): - self.assertEqual(old_reader_cursor[str(i)], expected) + self.assertEqual(old_reader_cursor[self.get_key(i)], expected) old_reader_session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index e02dbf99e09..b26b844dd32 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -31,6 +31,7 @@ import wiredtiger, wttest, re, suite_random from wtdataset import SimpleDataSet from contextlib import contextmanager +from wtscenario import make_scenarios class test_timestamp22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' @@ -46,6 +47,12 @@ class test_timestamp22(wttest.WiredTigerTestCase): SUCCESS = 'success' FAILURE = 'failure' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + # Control execution of an operation, looking for exceptions and error messages. # Usage: # with self.expect(self.FAILURE, 'some operation'): @@ -240,7 +247,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): msg = 'inserts with commit config(' + commit_config + ')' try: - for i in range(0, self.nrows): + for i in range(1, self.nrows + 1): needs_rollback = False if self.do_illegal(): # Illegal outside of transaction @@ -380,14 +387,14 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: iterations = 1000 - create_params = 'value_format=S,key_format=i' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(self.uri, create_params) self.set_global_timestamps(1, 1, -1, -1) # Create tables with no entries ds = SimpleDataSet( - self, self.uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, self.uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') # We do a bunch of iterations, doing transactions, prepare, and global timestamp calls # with timestamps that are sometimes valid, sometimes not. We use the iteration number @@ -430,7 +437,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Make sure the resulting rows are what we expect. cursor = self.session.open_cursor(self.uri) - expect_key = 0 + expect_key = 1 expect_value = self.commit_value for k,v in cursor: self.assertEquals(k, expect_key) @@ -440,7 +447,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Although it's theoretically possible to never successfully update a single row, # with a large number of iterations that should never happen. I'd rather catch # a test code error where we mistakenly don't update any rows. - self.assertGreater(expect_key, 0) + self.assertGreater(expect_key, 1) cursor.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp23.py b/src/third_party/wiredtiger/test/suite/test_timestamp23.py new file mode 100644 index 00000000000..60d97cf78fb --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp23.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_timestamp23.py +# +# delete keys repeatedly at successive timestamps +class test_timestamp23(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + + def test_timestamp(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:timestamp23' + ds = SimpleDataSet( + self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + key = 5 + value_1 = 'a' * 500 + value_2 = 'b' * 500 + value_3 = 'c' * 500 + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + cursor = self.session.open_cursor(ds.uri) + + # Write two values at timestamp 10. We'll muck with the first value + # and use the second to reference the page for eviction. + self.session.begin_transaction('read_timestamp=10') + cursor[key] = value_1 + cursor[key+1] = value_2 + self.session.commit_transaction('commit_timestamp=11') + + # Delete the first value at timestamp 20. + self.session.begin_transaction('read_timestamp=20') + cursor.set_key(key) + cursor.remove() + self.session.commit_transaction('commit_timestamp=21') + + # Put it back at timestamp 30. + self.session.begin_transaction('read_timestamp=30') + cursor[key] = value_3 + self.session.commit_transaction('commit_timestamp=31') + + # Delete it again at timestamp 40. + self.session.begin_transaction('read_timestamp=40') + cursor.set_key(key) + cursor.remove() + self.session.commit_transaction('commit_timestamp=41') + + # Evict the page using the second key. + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + v = evict_cursor[key+1] + self.assertEqual(v, value_2) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Create a separate session and a cursor to read the original value at timestamp 12. + session2 = self.conn.open_session() + cursor2 = session2.open_cursor(ds.uri) + session2.begin_transaction('read_timestamp=12') + v = cursor2[key] + self.assertEqual(v, value_1) + + self.session.breakpoint() + + # Now delete the original value. This _should_ cause WT_ROLLBACK, but with a column + # store bug seen and fixed in August 2021, it succeeds, and the resulting invalid + # tombstone will cause reconciliation to assert. (To see this behavior, comment out the + # self.fail call and let the transaction commit.) + try: + cursor2.remove() + self.fail("Conflicting remove did not fail") + session2.commit_transaction('commit_timestamp=50') + except wiredtiger.WiredTigerError as e: + self.assertTrue(wiredtiger.wiredtiger_strerror(wiredtiger.WT_ROLLBACK) in str(e)) + + cursor.close() + cursor2.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/test_coverage.md b/src/third_party/wiredtiger/test/test_coverage.md index bc71f581c09..b186c7ac318 100644 --- a/src/third_party/wiredtiger/test/test_coverage.md +++ b/src/third_party/wiredtiger/test/test_coverage.md @@ -53,4 +53,4 @@ |Truncate||[test_truncate01.py](../test/suite/test_truncate01.py) |Truncate|Prepare|[test_prepare13.py](../test/suite/test_prepare13.py) |Verify|Prepare|[test_prepare_hs03.py](../test/suite/test_prepare_hs03.py), [test_timestamp18.py](../test/suite/test_timestamp18.py) -|Wt Util||[test_backup01.py](../test/suite/test_backup01.py), [test_dump.py](../test/suite/test_dump.py), [test_util11.py](../test/suite/test_util11.py) +|Wt Util||[test_backup01.py](../test/suite/test_backup01.py), [test_dump.py](../test/suite/test_dump.py), [test_dump01.py](../test/suite/test_dump01.py), [test_util11.py](../test/suite/test_util11.py) |