diff options
author | Luke Chen <luke.chen@mongodb.com> | 2020-11-02 16:09:09 +1100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-11-02 05:25:04 +0000 |
commit | 03d22bb5884e280934d36702136d99a9363fb720 (patch) | |
tree | 6542aa97f47b8c2883d4cc1b36af9e4dea5863c2 | |
parent | 388a6a2435eba24ccc5da1da2c874543ddad6a7a (diff) | |
download | mongo-03d22bb5884e280934d36702136d99a9363fb720.tar.gz |
Import wiredtiger: d6d4b1cc014252aa4c99db8fddbbc9c5ddc3044f from branch mongodb-4.4
ref: aecb18daca..d6d4b1cc01
for: 4.4.3
WT-4310 Add option to not abort in diagnostic builds on data corruption
WT-4833 Drop support for Python2
WT-6712 Allow RTS to operate when stable timestamp is not set
WT-6809 Avoid deadlock by moving hs cursor caching earlier
WT-6811 Allow older readers to read behind a mixed-mode operation
WT-6822 Use the correct python for all tests and set up env for the split stress test
WT-6826 Disable python tests related to UTF decoding for OSX.
WT-6827 Run scons check examples in sequence not parallel
58 files changed, 1508 insertions, 992 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index fca0a520a65..ed1ef0e8351 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -9,6 +9,7 @@ import textwrap import distutils.sysconfig EnsureSConsVersion( 2, 0, 0 ) +EnsurePythonVersion(3, 0) if not os.sys.platform == "win32": print ("SConstruct is only supported for Windows, use build_posix for other platforms") @@ -60,6 +61,8 @@ var.Add('CPPPATH', 'C Preprocessor include path', [ "#/.", ]) +var.Add('LIBPATH', 'Adds paths to the linker search path', []) + var.Add('CFLAGS', 'C Compiler Flags', [ "/W3", # Warning level 3 "/WX", # Warnings are fatal @@ -310,7 +313,6 @@ if GetOption("lang-python"): print("The Python Interpreter must be 64-bit in order to build the python bindings") Exit(1) - pythonMajorVersion = sys.version_info.major pythonEnv = env.Clone() pythonEnv.Append(SWIGFLAGS=[ "-python", @@ -318,7 +320,6 @@ if GetOption("lang-python"): "-O", "-nodefaultctor", "-nodefaultdtor", - "-DPY_MAJOR_VERSION=" + str(pythonMajorVersion) ]) # Ignore warnings in swig-generated code. pythonEnv['CFLAGS'].remove("/WX") @@ -329,15 +330,22 @@ if GetOption("lang-python"): SHLIBSUFFIX=".pyd", LIBS=[wtlib] + wtlibs) - copySwig = pythonEnv.Command( - 'lang/python/wiredtiger/__init__.py', + # Shuffle the wiredtiger __init__ into place. + copySwig1 = pythonEnv.Command( + 'lang/python/wiredtiger/swig_wiredtiger.py', 'lang/python/wiredtiger.py', Copy('$TARGET', '$SOURCE')) - pythonEnv.Depends(copySwig, swiglib) + pythonEnv.Depends(copySwig1, swiglib) + + copySwig2 = pythonEnv.Command( + 'lang/python/wiredtiger/__init__.py', + 'lang/python/wiredtiger/init.py', + Copy('$TARGET', '$SOURCE')) + pythonEnv.Depends(copySwig2, swiglib) swiginstall = pythonEnv.Install('lang/python/wiredtiger/', swiglib) - Default(swiginstall, copySwig) + Default(swiginstall, copySwig1, copySwig2) # Shim library of functions to emulate POSIX on Windows shim = env.Library("window_shim", diff --git a/src/third_party/wiredtiger/build_posix/configure.ac.in b/src/third_party/wiredtiger/build_posix/configure.ac.in index 02c56f4fe73..3c427b45e14 100644 --- a/src/third_party/wiredtiger/build_posix/configure.ac.in +++ b/src/third_party/wiredtiger/build_posix/configure.ac.in @@ -149,7 +149,7 @@ if test "$wt_cv_enable_python" = "yes"; then fi if test "$wt_cv_enable_python" = "yes"; then - AM_PATH_PYTHON([2.6]) + AM_PATH_PYTHON([3.0]) if test -n "$with_python_prefix" ; then PYTHON_INSTALL_ARG="-d $with_python_prefix" fi diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 42a79ec006f..0df17bcfb1f 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -478,6 +478,9 @@ connection_runtime_config = [ Config('debug_mode', '', r''' control the settings of various extended debugging features''', type='category', subconfig=[ + Config('corruption_abort', 'true', r''' + if true, dump the core in the diagnostic mode on encountering the data corruption.''', + type='boolean'), Config('checkpoint_retention', '0', r''' adjust log archiving to retain the log records of this number of checkpoints. Zero or one means perform normal archiving.''', diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 3df400be17d..58090bf42be 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -13,6 +13,7 @@ API APIs ARG ARGS +ASAN ASM Addr Ailamaki diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 8e2d1ef2ca6..6d66318f99c 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -285,7 +285,6 @@ connection_stats = [ CacheStat('cache_hs_insert_restart', 'history store table insert calls that returned restart'), CacheStat('cache_hs_key_truncate', 'history store table truncation to remove an update'), CacheStat('cache_hs_key_truncate_mix_ts', 'history store table truncation to remove range of updates due to mixed timestamps'), - CacheStat('cache_hs_key_truncate_mix_ts_restart', 'history store table truncation due to mixed timestamps that returned restart'), CacheStat('cache_hs_key_truncate_onpage_removal', 'history store table truncation to remove range of updates due to key being removed from the data page during reconciliation'), CacheStat('cache_hs_key_truncate_rts', 'history store table truncation by rollback to stable to remove an update'), CacheStat('cache_hs_key_truncate_rts_unstable', 'history store table truncation by rollback to stable to remove an unstable update'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index b6a9a0ad3b6..4c20441ef39 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "aecb18daca54af24f50be193b62f85d35ad5e603" + "commit": "d6d4b1cc014252aa4c99db8fddbbc9c5ddc3044f" } diff --git a/src/third_party/wiredtiger/lang/python/Makefile.am b/src/third_party/wiredtiger/lang/python/Makefile.am index b15e66e0e83..cdbc615a6f1 100644 --- a/src/third_party/wiredtiger/lang/python/Makefile.am +++ b/src/third_party/wiredtiger/lang/python/Makefile.am @@ -3,8 +3,6 @@ PYDIRS = -t $(abs_builddir) -I $(abs_top_srcdir):$(abs_top_builddir) -L $(abs_to PYDST = $(abs_builddir)/wiredtiger PYFILES = $(PYDST)/fpacking.py $(PYDST)/intpacking.py $(PYDST)/packing.py \ $(PYDST)/packutil.py $(PYDST)/swig_wiredtiger.py $(PYDST)/__init__.py -PY_MAJOR_VERSION := $$($(PYTHON) -c \ - 'import sys; print(int(sys.version_info.major))') all-local: _wiredtiger.so pyfiles @@ -12,7 +10,7 @@ all-local: _wiredtiger.so pyfiles # in release packages. $(PYSRC)/wiredtiger_wrap.c $(PYSRC)/wiredtiger.py: $(top_srcdir)/src/include/wiredtiger.in $(PYSRC)/wiredtiger.i (cd $(PYSRC) && \ - $(SWIG) -python -DPY_MAJOR_VERSION=$(PY_MAJOR_VERSION) \ + $(SWIG) -python \ -threads -O -Wall -nodefaultctor -nodefaultdtor \ -I$(abs_top_builddir) wiredtiger.i) diff --git a/src/third_party/wiredtiger/lang/python/run-ex_access b/src/third_party/wiredtiger/lang/python/run-ex_access index 8e3f09b9cfe..439451b70eb 100755 --- a/src/third_party/wiredtiger/lang/python/run-ex_access +++ b/src/third_party/wiredtiger/lang/python/run-ex_access @@ -1,5 +1,6 @@ #!/bin/sh +PYTHON=${PYTHON:-python3} rm -rf WT_TEST ; mkdir WT_TEST -exec env LD_LIBRARY_PATH=../../.libs DYLD_LIBRARY_PATH=../../.libs PYTHONPATH=.:${srcdir} python -S ${srcdir}/../../examples/python/ex_access.py +exec env LD_LIBRARY_PATH=../../.libs DYLD_LIBRARY_PATH=../../.libs PYTHONPATH=.:${srcdir} ${PYTHON} -S ${srcdir}/../../examples/python/ex_access.py diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index 122057cca12..b71fb7cd299 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -251,7 +251,7 @@ from packing import pack, unpack } /* Internal _set_key, _set_value methods take a 'bytes' object as parameter. */ -%pybuffer_binary(void *data, int); +%pybuffer_binary(unsigned char *data, int); /* Throw away references after close. */ %define DESTRUCTOR(class, method) @@ -548,11 +548,7 @@ OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, equals, (self, other)) OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, search_near, (self)) /* SWIG magic to turn Python byte strings into data / size. */ -#if PY_MAJOR_VERSION >= 3 - %apply (char *STRING, int LENGTH) { (char *data, int size) }; -#else -%apply (char *STRING, int LENGTH) { (void *data, int size) }; -#endif +%apply (char *STRING, int LENGTH) { (char *data, int size) }; /* Handle binary data returns from get_key/value -- avoid cstring.i: it creates a list of returns. */ %typemap(in,numinputs=0) (char **datap, int *sizep) (char *data, int size) { $1 = &data; $2 = &size; } @@ -588,7 +584,7 @@ typedef int int_void; %extend __wt_cursor { /* Get / set keys and values */ - void _set_key(void *data, int size) { + void _set_key(unsigned char *data, int size) { WT_ITEM k; k.data = data; k.size = (uint32_t)size; @@ -617,7 +613,7 @@ typedef int int_void; return (ret); } - void _set_value(void *data, int size) { + void _set_value(unsigned char *data, int size) { WT_ITEM v; v.data = data; v.size = (uint32_t)size; diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/init.py b/src/third_party/wiredtiger/lang/python/wiredtiger/init.py index 6dbc4d7a35c..09bd9ed18a2 100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/init.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/init.py @@ -38,6 +38,10 @@ if fname != '__init__.py' and fname != '__init__.pyc': print(__file__ + ': this file is not yet installed') sys.exit(1) +if sys.version_info[0] <= 2: + print('WiredTiger requires Python version 3.0 or above') + sys.exit(1) + # After importing the SWIG-generated file, copy all symbols from from it # to this module so they will appear in the wiredtiger namespace. me = sys.modules[__name__] diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py b/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py index c6032354de6..e001bb552d2 100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/intpacking.py @@ -29,11 +29,7 @@ from __future__ import print_function import math, struct, sys -try: - from wiredtiger.packutil import _chr, _ord, x00_entry, xff_entry -except ImportError: - # When WiredTiger is installed as a package, python2 needs this - from .packutil import _chr, _ord, x00_entry, xff_entry +from wiredtiger.packutil import _chr, _ord, x00_entry, xff_entry # Variable-length integer packing # need: up to 64 bits, both signed and unsigned diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py index 288014186b6..83821f2657f 100755 --- a/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/packing.py @@ -49,15 +49,9 @@ Format Python Notes u str raw byte array """ -try: - from wiredtiger.packutil import _chr, _is_string, _ord, _string_result, \ - empty_pack, x00 - from wiredtiger.intpacking import pack_int, unpack_int -except ImportError: - # When WiredTiger is installed as a package, python2 needs this - from .packutil import _chr, _is_string, _ord, _string_result, \ - empty_pack, x00 - from intpacking import pack_int, unpack_int +from wiredtiger.packutil import _chr, _is_string, _ord, _string_result, \ + empty_pack, x00 +from wiredtiger.intpacking import pack_int, unpack_int def __get_type(fmt): if not fmt: diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index cfdd1664fd4..20568ce0941 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -384,7 +384,12 @@ __desc_read(WT_SESSION_IMPL *session, uint32_t allocsize, WT_BLOCK *block) */ if (F_ISSET(session, WT_SESSION_IMPORT_REPAIR)) goto err; - WT_ERR_MSG(session, WT_ERROR, "%s does not appear to be a WiredTiger file", block->name); + + if (F_ISSET(session, WT_SESSION_ROLLBACK_TO_STABLE)) + ret = ENOENT; + else + WT_ERR_MSG( + session, WT_ERROR, "%s does not appear to be a WiredTiger file", block->name); } if (desc->majorv > WT_BLOCK_MAJOR_VERSION || diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index e5521bf1a70..233d8ce6889 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -71,6 +71,7 @@ __wt_delete_page(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) return (0); } + WT_RET(__wt_hs_cursor_cache(session)); (void)__wt_atomic_addv32(&S2BT(session)->evict_busy, 1); ret = __wt_evict(session, ref, previous_state, 0); (void)__wt_atomic_subv32(&S2BT(session)->evict_busy, 1); diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index a0d9c00a014..69a33d7ac6a 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -42,6 +42,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_compatibility_sub static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = { {"checkpoint_retention", "int", NULL, "min=0,max=1024", NULL, 0}, + {"corruption_abort", "boolean", NULL, NULL, NULL, 0}, {"cursor_copy", "boolean", NULL, NULL, NULL, 0}, {"eviction", "boolean", NULL, NULL, NULL, 0}, {"log_retention", "int", NULL, "min=0,max=1024", NULL, 0}, {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, @@ -96,7 +97,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"compatibility", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, {"error_prefix", "string", NULL, NULL, NULL, 0}, {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2}, {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0}, @@ -541,7 +542,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -617,7 +618,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -692,7 +693,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -763,7 +764,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -844,13 +845,14 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_CONNECTION.reconfigure", "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),compatibility=(release=)," - "debug_mode=(checkpoint_retention=0,cursor_copy=false," - "eviction=false,log_retention=0,realloc_exact=false," - "rollback_error=0,slow_checkpoint=false,table_logging=false)," - "error_prefix=,eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",eviction_updates_target=0,eviction_updates_trigger=0," + "debug_mode=(checkpoint_retention=0,corruption_abort=true," + "cursor_copy=false,eviction=false,log_retention=0," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),error_prefix=,eviction=(threads_max=8," + "threads_min=1),eviction_checkpoint_target=1," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,eviction_updates_target=0" + ",eviction_updates_trigger=0," "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),history_store=(file_max=0)," "io_capacity=(total=0),log=(archive=true,os_cache_dirty_pct=0," @@ -1022,11 +1024,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," "compatibility=(release=,require_max=,require_min=)," "config_base=true,create=false,debug_mode=(checkpoint_retention=0" - ",cursor_copy=false,eviction=false,log_retention=0," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + ",corruption_abort=true,cursor_copy=false,eviction=false," + "log_retention=0,realloc_exact=false,rollback_error=0," + "slow_checkpoint=false,table_logging=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," @@ -1053,11 +1056,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," "compatibility=(release=,require_max=,require_min=)," "config_base=true,create=false,debug_mode=(checkpoint_retention=0" - ",cursor_copy=false,eviction=false,log_retention=0," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + ",corruption_abort=true,cursor_copy=false,eviction=false," + "log_retention=0,realloc_exact=false,rollback_error=0," + "slow_checkpoint=false,table_logging=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," @@ -1084,12 +1088,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," "compatibility=(release=,require_max=,require_min=)," - "debug_mode=(checkpoint_retention=0,cursor_copy=false," - "eviction=false,log_retention=0,realloc_exact=false," - "rollback_error=0,slow_checkpoint=false,table_logging=false)," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "debug_mode=(checkpoint_retention=0,corruption_abort=true," + "cursor_copy=false,eviction=false,log_retention=0," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," @@ -1115,12 +1119,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," "compatibility=(release=,require_max=,require_min=)," - "debug_mode=(checkpoint_retention=0,cursor_copy=false," - "eviction=false,log_retention=0,realloc_exact=false," - "rollback_error=0,slow_checkpoint=false,table_logging=false)," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "debug_mode=(checkpoint_retention=0,corruption_abort=true," + "cursor_copy=false,eviction=false,log_retention=0," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 99ec1ed72cb..bc2ad006ab6 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1789,6 +1789,12 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) */ WT_PUBLISH(conn->debug_ckpt_cnt, (uint32_t)cval.val); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.corruption_abort", &cval)); + if (cval.val) + FLD_SET(conn->debug_flags, WT_CONN_DEBUG_CORRUPTION_ABORT); + else + FLD_CLR(conn->debug_flags, WT_CONN_DEBUG_CORRUPTION_ABORT); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.cursor_copy", &cval)); if (cval.val) FLD_SET(conn->debug_flags, WT_CONN_DEBUG_CURSOR_COPY); diff --git a/src/third_party/wiredtiger/src/cursor/cur_hs.c b/src/third_party/wiredtiger/src/cursor/cur_hs.c index 8aaca515f11..ba2799e2127 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_hs.c +++ b/src/third_party/wiredtiger/src/cursor/cur_hs.c @@ -10,19 +10,16 @@ #include "wt_internal.h" /* - * __wt_hs_cursor_open -- - * Open a new history store table cursor. + * __hs_cursor_open_int -- + * Open a new history store table cursor, internal function. */ -int -__wt_hs_cursor_open(WT_SESSION_IMPL *session) +static int +__hs_cursor_open_int(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) { WT_CURSOR *cursor; WT_DECL_RET; const char *open_cursor_cfg[] = {WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL}; - /* Not allowed to open a cursor if you already have one */ - WT_ASSERT(session, session->hs_cursor == NULL); - WT_WITHOUT_DHANDLE( session, ret = __wt_open_cursor(session, WT_HS_URI, NULL, open_cursor_cfg, &cursor)); WT_RET(ret); @@ -30,11 +27,66 @@ __wt_hs_cursor_open(WT_SESSION_IMPL *session) /* History store cursors should always ignore tombstones. */ F_SET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); - session->hs_cursor = cursor; + *cursorp = cursor; return (0); } /* + * __wt_hs_cursor_cache -- + * Cache a new history store table cursor. Open and then close a history store cursor without + * saving it in the session. + */ +int +__wt_hs_cursor_cache(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_CURSOR *cursor; + + conn = S2C(session); + + /* + * Make sure this session has a cached history store cursor, otherwise we can deadlock with a + * session wanting exclusive access to a handle: that session will have a handle list write lock + * and will be waiting on eviction to drain, we'll be inside eviction waiting on a handle list + * read lock to open a history store cursor. + * + * The test for the no-reconciliation flag is necessary because the session may already be doing + * history store operations and if we open/close the existing history store cursor, we can + * affect those already-running history store operations by changing the cursor state. When + * doing history store operations, we set the no-reconciliation flag, use it as short-hand to + * avoid that problem. This doesn't open up the window for the deadlock because setting the + * no-reconciliation flag limits eviction to in-memory splits. + * + * The test for the connection's default session is because there are known problems with using + * cached cursors from the default session. The metadata does not have history store content and + * is commonly handled specially. We won't need a history store cursor if we are evicting + * metadata. + * + * FIXME-WT-6037: This isn't reasonable and needs a better fix. + */ + if (F_ISSET(conn, WT_CONN_IN_MEMORY) || F_ISSET(session, WT_SESSION_NO_RECONCILE) || + (session->dhandle != NULL && WT_IS_METADATA(S2BT(session)->dhandle)) || + session == conn->default_session) + return (0); + WT_RET(__hs_cursor_open_int(session, &cursor)); + WT_RET(cursor->close(cursor)); + return (0); +} + +/* + * __wt_hs_cursor_open -- + * Open a new history store table cursor wrapper function. + */ +int +__wt_hs_cursor_open(WT_SESSION_IMPL *session) +{ + /* Not allowed to open a cursor if you already have one */ + WT_ASSERT(session, session->hs_cursor == NULL); + + return (__hs_cursor_open_int(session, &session->hs_cursor)); +} + +/* * __wt_hs_cursor_close -- * Discard a history store cursor. */ diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 0d334c2216a..f3cf48e3c67 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -282,11 +282,7 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) * busy and then opens a different file (in this case, the HS file), it can deadlock with a * thread waiting for the first file to drain from the eviction queue. See WT-5946 for details. */ - if (session->hs_cursor == NULL && !F_ISSET(conn, WT_CONN_IN_MEMORY)) { - WT_RET(__wt_hs_cursor_open(session)); - WT_RET(__wt_hs_cursor_close(session)); - } - + WT_RET(__wt_hs_cursor_cache(session)); if (conn->evict_server_running && __wt_spin_trylock(session, &cache->evict_pass_lock) == 0) { /* * Cannot use WT_WITH_PASS_LOCK because this is a try lock. Fix when that is supported. We @@ -2298,6 +2294,7 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d { WT_CACHE *cache; WT_CONNECTION_IMPL *conn; + WT_CURSOR *hs_cursor_saved; WT_DECL_RET; WT_TRACK_OP_DECL; WT_TXN_GLOBAL *txn_global; @@ -2308,6 +2305,7 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d WT_TRACK_OP_INIT(session); + app_thread = false; conn = S2C(session); cache = conn->cache; time_start = time_stop = 0; @@ -2315,6 +2313,23 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d txn_shared = WT_SESSION_TXN_SHARED(session); /* + * If we have a history store cursor, save it. This ensures that if eviction needs to access the + * history store, it will get its own cursor, avoiding potential problems if it were to + * reposition or reset a history store cursor that we're in the middle of using for something + * else. + */ + hs_cursor_saved = session->hs_cursor; + session->hs_cursor = NULL; + + /* + * Before we enter the eviction generation, make sure this session has a cached history store + * cursor, otherwise we can deadlock with a session wanting exclusive access to a handle: that + * session will have a handle list write lock and will be waiting on eviction to drain, we'll be + * inside eviction waiting on a handle list read lock to open a history store cursor. + */ + WT_ERR(__wt_hs_cursor_cache(session)); + + /* * It is not safe to proceed if the eviction server threads aren't setup yet. */ if (!conn->evict_server_running || (busy && pct_full < 100.0)) @@ -2412,6 +2427,13 @@ err: done: WT_TRACK_OP_END(session); + + /* If the caller was using a history store cursor they should have closed it by now. */ + WT_ASSERT(session, session->hs_cursor == NULL); + + /* Restore the caller's history store cursor. */ + session->hs_cursor = hs_cursor_saved; + return (ret); } diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 3195928ac77..9d926e28946 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -77,6 +77,7 @@ __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) evict_flags = LF_ISSET(WT_READ_NO_SPLIT) ? WT_EVICT_CALL_NO_SPLIT : 0; FLD_SET(evict_flags, WT_EVICT_CALL_URGENT); + WT_RET(__wt_hs_cursor_cache(session)); (void)__wt_atomic_addv32(&btree->evict_busy, 1); ret = __wt_evict(session, ref, previous_state, evict_flags); (void)__wt_atomic_subv32(&btree->evict_busy, 1); @@ -92,7 +93,6 @@ int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32_t flags) { WT_CONNECTION_IMPL *conn; - WT_CURSOR *hs_cursor_saved; WT_DECL_RET; WT_PAGE *page; uint64_t time_start, time_stop; @@ -108,45 +108,11 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32 __wt_verbose( session, WT_VERB_EVICT, "page %p (%s)", (void *)page, __wt_page_type_string(page->type)); - /* - * If we have a history store cursor, save it. This ensures that if eviction needs to access the - * history store, it will get its own cursor, avoiding potential problems if it were to - * reposition or reset a history store cursor that we're in the middle of using for something - * else. - */ - hs_cursor_saved = session->hs_cursor; - session->hs_cursor = NULL; - tree_dead = F_ISSET(session->dhandle, WT_DHANDLE_DEAD); if (tree_dead) LF_SET(WT_EVICT_CALL_NO_SPLIT); /* - * Before we enter the eviction generation, make sure this session has a cached history store - * cursor, otherwise we can deadlock with a session wanting exclusive access to a handle: that - * session will have a handle list write lock and will be waiting on eviction to drain, we'll be - * inside eviction waiting on a handle list read lock to open a history store cursor. - * - * The test for the no-reconciliation flag is necessary because the session may already be doing - * history store operations and if we open/close the existing history store cursor, we can - * affect those already-running history store operations by changing the cursor state. When - * doing history store operations, we set the no-reconciliation flag, use it as short-hand to - * avoid that problem. This doesn't open up the window for the deadlock because setting the - * no-reconciliation flag limits eviction to in-memory splits. - * - * The test for the connection's default session is because there are known problems with using - * cached cursors from the default session. - * - * FIXME-WT-6037: This isn't reasonable and needs a better fix. - */ - if (!WT_IS_METADATA(S2BT(session)->dhandle) && !F_ISSET(conn, WT_CONN_IN_MEMORY) && - session->hs_cursor == NULL && !F_ISSET(session, WT_SESSION_NO_RECONCILE) && - session != conn->default_session) { - WT_RET(__wt_hs_cursor_open(session)); - WT_RET(__wt_hs_cursor_close(session)); - } - - /* * Enter the eviction generation. If we re-enter eviction, leave the previous eviction * generation (which must be as low as the current generation), untouched. */ @@ -286,12 +252,6 @@ done: if (local_gen) __wt_session_gen_leave(session, WT_GEN_EVICT); - /* If the caller was using a history store cursor they should have closed it by now. */ - WT_ASSERT(session, session->hs_cursor == NULL); - - /* Restore caller's history store cursor. */ - session->hs_cursor = hs_cursor_saved; - return (ret); } diff --git a/src/third_party/wiredtiger/src/history/hs_cursor.c b/src/third_party/wiredtiger/src/history/hs_cursor.c index 02de1fc5a52..aa3f4aea5cb 100644 --- a/src/third_party/wiredtiger/src/history/hs_cursor.c +++ b/src/third_party/wiredtiger/src/history/hs_cursor.c @@ -106,8 +106,7 @@ __hs_cursor_position_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, uint32_t b * history store as opposed to comparing the embedded data store key since the ordering is not * guaranteed to be the same. */ - cursor->set_key( - cursor, btree_id, key, timestamp != WT_TS_NONE ? timestamp : WT_TS_MAX, UINT64_MAX); + cursor->set_key(cursor, btree_id, key, timestamp, UINT64_MAX); /* Copy the raw key before searching as a basis for comparison. */ WT_ERR(__wt_buf_set(session, srch_key, cursor->key.data, cursor->key.size)); WT_ERR(cursor->search_near(cursor, &exact)); @@ -225,6 +224,15 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma * history store) to the oldest (earlier in the history store) for a given key. */ read_timestamp = allow_prepare ? txn->prepare_timestamp : txn_shared->read_timestamp; + + /* + * A reader without a timestamp should read the largest timestamp in the range, however cursor + * search near if given a 0 timestamp will place at the top of the range and hide the records + * below it. As such we need to adjust a 0 timestamp to the timestamp max value. + */ + if (read_timestamp == WT_TS_NONE) + read_timestamp = WT_TS_MAX; + WT_ERR_NOTFOUND_OK( __wt_hs_cursor_position(session, hs_cursor, hs_btree_id, key, read_timestamp, NULL), true); if (ret == WT_NOTFOUND) { diff --git a/src/third_party/wiredtiger/src/history/hs_rec.c b/src/third_party/wiredtiger/src/history/hs_rec.c index 2c5e48e1ca6..08856b361a2 100644 --- a/src/third_party/wiredtiger/src/history/hs_rec.c +++ b/src/third_party/wiredtiger/src/history/hs_rec.c @@ -18,8 +18,8 @@ typedef struct { uint64_t txnid; } WT_HS_TIME_POINT; -static int __hs_delete_key_from_pos( - WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, uint32_t btree_id, const WT_ITEM *key); +static int __hs_delete_key_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, + uint32_t btree_id, const WT_ITEM *key, bool reinsert); static int __hs_fixup_out_of_order_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_BTREE *btree, const WT_ITEM *key, wt_timestamp_t ts, uint64_t *hs_counter, const WT_ITEM *srch_key); @@ -76,7 +76,7 @@ __hs_insert_updates_verbose(WT_SESSION_IMPL *session, WT_BTREE *btree) * parameter will be positioned on the newly inserted record. Otherwise, it will be reset. */ static int -__hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, +__hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, uint64_t btree_id, const WT_ITEM *key, const uint8_t type, const WT_ITEM *hs_value, WT_HS_TIME_POINT *start_time_point, WT_HS_TIME_POINT *stop_time_point, uint64_t counter) { @@ -91,7 +91,7 @@ __hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, W * Use WT_CURSOR.set_key and WT_CURSOR.set_value to create key and value items, then use them to * create an update chain for a direct insertion onto the history store page. */ - cursor->set_key(cursor, btree->id, key, start_time_point->ts, counter); + cursor->set_key(cursor, btree_id, key, start_time_point->ts, counter); cursor->set_value( cursor, stop_time_point->durable_ts, start_time_point->durable_ts, (uint64_t)type, hs_value); @@ -156,13 +156,12 @@ err: */ static int __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, - const WT_ITEM *key, const WT_UPDATE *upd, const uint8_t type, const WT_ITEM *hs_value, - WT_HS_TIME_POINT *stop_time_point, bool clear_hs) + const WT_ITEM *key, const uint8_t type, const WT_ITEM *hs_value, + WT_HS_TIME_POINT *start_time_point, WT_HS_TIME_POINT *stop_time_point) { WT_DECL_ITEM(hs_key); WT_DECL_ITEM(srch_key); WT_DECL_RET; - WT_HS_TIME_POINT start_time_point; wt_timestamp_t hs_start_ts; uint64_t counter, hs_counter; uint32_t hs_btree_id; @@ -201,7 +200,8 @@ __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BT * one can lead to wrong order. */ WT_ERR_NOTFOUND_OK( - __wt_hs_cursor_position(session, cursor, btree->id, key, upd->start_ts, srch_key), true); + __wt_hs_cursor_position(session, cursor, btree->id, key, start_time_point->ts, srch_key), + true); if (ret == 0) { WT_ERR(cursor->get_key(cursor, &hs_btree_id, hs_key, &hs_start_ts, &hs_counter)); @@ -211,7 +211,7 @@ __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BT * Verify simple checks first to confirm whether the retrieved update same or not before * performing the expensive key comparison. */ - if (hs_btree_id == btree->id && upd->start_ts == hs_start_ts) { + if (hs_btree_id == btree->id && start_time_point->ts == hs_start_ts) { WT_ERR(__wt_compare(session, NULL, hs_key, key, &cmp)); if (cmp == 0) counter = hs_counter + 1; @@ -222,52 +222,17 @@ __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BT * If we're inserting a non-zero timestamp, look ahead for any higher timestamps. If we find * updates, we should remove them and reinsert them at the current timestamp. */ - if (upd->start_ts != WT_TS_NONE) { + if (start_time_point->ts != WT_TS_NONE) { WT_ERR_NOTFOUND_OK(__wt_hs_cursor_next(session, cursor), true); if (ret == 0) WT_ERR(__hs_fixup_out_of_order_from_pos( - session, cursor, btree, key, upd->start_ts, &counter, srch_key)); + session, cursor, btree, key, start_time_point->ts, &counter, srch_key)); } - start_time_point.ts = upd->start_ts; - start_time_point.durable_ts = upd->durable_ts; - start_time_point.txnid = upd->txnid; - /* The tree structure can change while we try to insert the mod list, retry if that happens. */ - while ((ret = __hs_insert_record_with_btree_int(session, cursor, btree, key, type, hs_value, - &start_time_point, stop_time_point, counter)) == WT_RESTART) + while ((ret = __hs_insert_record_with_btree_int(session, cursor, btree->id, key, type, hs_value, + start_time_point, stop_time_point, counter)) == WT_RESTART) WT_STAT_CONN_INCR(session, cache_hs_insert_restart); - WT_ERR(ret); - - /* Done if we don't need to clear the history store content. */ - if (!clear_hs) - goto done; - - /* - * We can only insert update without timestamp into the history store if we need to clear the - * history store record. - */ - WT_ASSERT(session, upd->start_ts == WT_TS_NONE); - - /* - * If we need to clear the history store content, we need to delete all history records for that - * key that are further in the history table than us (the key is lexicographically greater). For - * timestamped tables that are occasionally getting a non-timestamped update, that means that - * all timestamped updates should get removed. - */ - WT_ERR_NOTFOUND_OK(__wt_hs_cursor_next(session, cursor), true); - - /* No records to delete. */ - if (ret == WT_NOTFOUND) { - ret = 0; - goto done; - } - while ((ret = __hs_delete_key_from_pos(session, cursor, btree->id, key)) == WT_RESTART) - WT_STAT_CONN_INCR(session, cache_hs_key_truncate_mix_ts_restart); - WT_ERR(ret); - WT_STAT_CONN_INCR(session, cache_hs_key_truncate_mix_ts); - -done: err: __wt_scr_free(session, &hs_key); __wt_scr_free(session, &srch_key); @@ -283,8 +248,8 @@ err: */ static int __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, const WT_ITEM *key, - const WT_UPDATE *upd, const uint8_t type, const WT_ITEM *hs_value, - WT_HS_TIME_POINT *stop_time_point, bool clear_hs) + const uint8_t type, const WT_ITEM *hs_value, WT_HS_TIME_POINT *start_time_point, + WT_HS_TIME_POINT *stop_time_point) { WT_CURSOR_BTREE *cbt; WT_DECL_RET; @@ -292,7 +257,7 @@ __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, cbt = (WT_CURSOR_BTREE *)cursor; WT_WITH_BTREE(session, CUR2BT(cbt), ret = __hs_insert_record_with_btree( - session, cursor, btree, key, upd, type, hs_value, stop_time_point, clear_hs)); + session, cursor, btree, key, type, hs_value, start_time_point, stop_time_point)); return (ret); } @@ -353,7 +318,7 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) WT_SAVE_UPD *list; WT_UPDATE *first_globally_visible_upd, *first_non_ts_upd; WT_UPDATE *non_aborted_upd, *oldest_upd, *prev_upd, *tombstone, *upd; - WT_HS_TIME_POINT stop_time_point; + WT_HS_TIME_POINT start_time_point, stop_time_point; wt_off_t hs_size; wt_timestamp_t min_insert_ts; uint64_t insert_cnt, max_hs_size; @@ -361,8 +326,7 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) uint8_t *p; int nentries; char ts_string[3][WT_TS_INT_STRING_SIZE]; - bool clear_hs, enable_reverse_modify, hs_inserted, squashed, ts_updates_in_hs; - + bool enable_reverse_modify, hs_inserted, squashed, ts_updates_in_hs; btree = S2BT(session); cursor = session->hs_cursor; prev_upd = NULL; @@ -491,8 +455,6 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) /* Track the first update that is globally visible. */ if (first_globally_visible_upd == NULL && __wt_txn_upd_visible_all(session, upd)) first_globally_visible_upd = upd; - else if (first_globally_visible_upd != NULL) - F_SET(upd, WT_UPDATE_OBSOLETE); /* * Always insert full update to the history store if we write a prepared update to the @@ -514,11 +476,7 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) if (first_non_ts_upd == NULL && upd->start_ts == WT_TS_NONE) first_non_ts_upd = upd; else if (first_non_ts_upd != NULL && upd->start_ts != WT_TS_NONE) { - /* - * Don't insert updates with timestamps after updates without timestamps to the - * history store. - */ - F_SET(upd, WT_UPDATE_OBSOLETE); + F_SET(upd, WT_UPDATE_BEHIND_MIXED_MODE); if (F_ISSET(upd, WT_UPDATE_HS)) ts_updates_in_hs = true; } @@ -557,22 +515,15 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) if (oldest_upd->type == WT_UPDATE_TOMBSTONE && oldest_upd == first_non_ts_upd && !F_ISSET(first_non_ts_upd, WT_UPDATE_CLEARED_HS)) { /* We can only delete history store entries that have timestamps. */ - WT_ERR(__wt_hs_delete_key_from_ts(session, btree->id, key, 1)); + WT_ERR(__wt_hs_delete_key_from_ts(session, btree->id, key, 1, true)); WT_STAT_CONN_INCR(session, cache_hs_key_truncate_mix_ts); - clear_hs = false; F_SET(first_non_ts_upd, WT_UPDATE_CLEARED_HS); - } else - /* - * Clear the content with timestamps in the history store if we see updates without - * timestamps on the update chain. - * - * We don't need to clear the history store records if everything is still on the insert - * list and there are no updates moved to the history store by checkpoint or a failed - * eviction. - */ - clear_hs = first_non_ts_upd != NULL && - !F_ISSET(first_non_ts_upd, WT_UPDATE_CLEARED_HS) && - (list->ins == NULL || ts_updates_in_hs); + } else if (first_non_ts_upd != NULL && !F_ISSET(first_non_ts_upd, WT_UPDATE_CLEARED_HS) && + (list->ins == NULL || ts_updates_in_hs)) { + WT_ERR(__wt_hs_delete_key_from_ts(session, btree->id, key, 1, true)); + WT_STAT_CONN_INCR(session, cache_hs_key_truncate_mix_ts); + F_SET(first_non_ts_upd, WT_UPDATE_CLEARED_HS); + } WT_ERR(__hs_next_upd_full_value(session, &modifies, NULL, full_value, &upd)); @@ -589,6 +540,9 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) upd = prev_upd) { WT_ASSERT(session, upd->type == WT_UPDATE_STANDARD || upd->type == WT_UPDATE_MODIFY); + start_time_point.durable_ts = upd->durable_ts; + start_time_point.ts = upd->start_ts; + start_time_point.txnid = upd->txnid; tombstone = NULL; __wt_modify_vector_peek(&modifies, &prev_upd); @@ -636,19 +590,27 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) continue; } - /* Skip updates that are obsolete. */ - if (F_ISSET(upd, WT_UPDATE_OBSOLETE)) - continue; + /* + * When we see an update older than a mixed mode update we need to insert it with a zero + * start and stop timestamp. This means it'll still exist but only use txnid visibility + * rules. As such older readers should still be able to see it. + */ + if (F_ISSET(upd, WT_UPDATE_BEHIND_MIXED_MODE)) { + start_time_point.ts = start_time_point.durable_ts = WT_TS_NONE; + stop_time_point.ts = stop_time_point.durable_ts = WT_TS_NONE; + } /* * If the time points are out of order (which can happen if the application performs * updates with out-of-order timestamps), so this value can never be seen, don't bother - * inserting it. + * inserting it. However if it was made obsolete by a mixed mode operation we still want + * to insert it, it will be flagged as such. * * FIXME-WT-6443: We should be able to replace this with an assertion. */ - if (stop_time_point.ts < upd->start_ts || - (stop_time_point.ts == upd->start_ts && stop_time_point.txnid <= upd->txnid)) { + if (!F_ISSET(upd, WT_UPDATE_BEHIND_MIXED_MODE) && + (stop_time_point.ts < upd->start_ts || + (stop_time_point.ts == upd->start_ts && stop_time_point.txnid <= upd->txnid))) { __wt_verbose(session, WT_VERB_TIMESTAMP, "Warning: fixing out-of-order timestamps %s earlier than previous update %s", __wt_timestamp_to_string(stop_time_point.ts, ts_string[0]), @@ -657,19 +619,6 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) } /* - * Clear history store content if we skip inserting the updates without timestamp. e.g., - * if we have an update chain U@30 -> M@20 -> U@0 and M@20 is globally visible, we skip - * writing U@0 to the history store. - */ - if (clear_hs && upd->start_ts != WT_TS_NONE) { - /* We can only delete history store entries that have timestamps. */ - WT_ERR(__wt_hs_delete_key_from_ts(session, btree->id, key, 1)); - WT_STAT_CONN_INCR(session, cache_hs_key_truncate_mix_ts); - clear_hs = false; - F_SET(first_non_ts_upd, WT_UPDATE_CLEARED_HS); - } - - /* * Calculate reverse modify and clear the history store records with timestamps when * inserting the first update. */ @@ -678,17 +627,13 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) __wt_calc_modify(session, prev_full_value, full_value, prev_full_value->size / 10, entries, &nentries) == 0) { WT_ERR(__wt_modify_pack(cursor, entries, nentries, &modify_value)); - WT_ERR(__hs_insert_record(session, cursor, btree, key, upd, WT_UPDATE_MODIFY, - modify_value, &stop_time_point, clear_hs)); + WT_ERR(__hs_insert_record(session, cursor, btree, key, WT_UPDATE_MODIFY, + modify_value, &start_time_point, &stop_time_point)); __wt_scr_free(session, &modify_value); } else - WT_ERR(__hs_insert_record(session, cursor, btree, key, upd, WT_UPDATE_STANDARD, - full_value, &stop_time_point, clear_hs)); - - if (clear_hs) - F_SET(first_non_ts_upd, WT_UPDATE_CLEARED_HS); + WT_ERR(__hs_insert_record(session, cursor, btree, key, WT_UPDATE_STANDARD, + full_value, &start_time_point, &stop_time_point)); - clear_hs = false; /* Flag the update as now in the history store. */ F_SET(upd, WT_UPDATE_HS); if (tombstone != NULL) @@ -703,30 +648,6 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) if (modifies.size > 0) WT_STAT_CONN_INCR(session, cache_hs_write_squash); - - /* - * We need to clear the history store if we haven't inserted anything into the history store - * and there are updates without timestamps in the middle of the update chain. - * - * e.g., U@10 -> T@0 -> U@5. - * - * But we don't need to clear the history store if we write an update without timestamp to - * the data store because we don't insert any update with timestamp to the history store and - * we will clear the history store again once that update is moved to the history store. - * - * e.g., U@0 -> U@10 -> U@5 and U@1 in the history store. U@10 and U@5 are not inserted to - * the history store as they are flagged as WT_UPDATE_MASKED_BY_NON_TS_UPDATE and U@1 is not - * removed from the history store. U@1 will be removed from the history store once U@0 is - * moved to the history store. - */ - if (clear_hs && - (first_non_ts_upd->txnid != list->onpage_upd->txnid || - first_non_ts_upd->start_ts != list->onpage_upd->start_ts)) { - /* We can only delete history store entries that have timestamps. */ - WT_ERR(__wt_hs_delete_key_from_ts(session, btree->id, key, 1)); - WT_STAT_CONN_INCR(session, cache_hs_key_truncate_mix_ts); - F_SET(first_non_ts_upd, WT_UPDATE_CLEARED_HS); - } } WT_ERR(__wt_block_manager_named_size(session, WT_HS_FILE, &hs_size)); @@ -757,7 +678,7 @@ err: */ static int __hs_delete_key_from_ts_int( - WT_SESSION_IMPL *session, uint32_t btree_id, const WT_ITEM *key, wt_timestamp_t ts) + WT_SESSION_IMPL *session, uint32_t btree_id, const WT_ITEM *key, wt_timestamp_t ts, bool reinsert) { WT_CURSOR *hs_cursor; WT_DECL_ITEM(srch_key); @@ -805,7 +726,7 @@ __hs_delete_key_from_ts_int( goto done; WT_ASSERT(session, ts == WT_TS_NONE || hs_start_ts != WT_TS_NONE); - WT_ERR(__hs_delete_key_from_pos(session, hs_cursor, btree_id, key)); + WT_ERR(__hs_delete_key_from_pos(session, hs_cursor, btree_id, key, reinsert)); done: ret = 0; err: @@ -819,7 +740,7 @@ err: */ int __wt_hs_delete_key_from_ts( - WT_SESSION_IMPL *session, uint32_t btree_id, const WT_ITEM *key, wt_timestamp_t ts) + WT_SESSION_IMPL *session, uint32_t btree_id, const WT_ITEM *key, wt_timestamp_t ts, bool reinsert) { WT_DECL_RET; @@ -827,7 +748,7 @@ __wt_hs_delete_key_from_ts( WT_ASSERT(session, !F_ISSET(session, WT_SESSION_NO_DATA_HANDLES)); /* The tree structure can change while we try to insert the mod list, retry if that happens. */ - while ((ret = __hs_delete_key_from_ts_int(session, btree_id, key, ts)) == WT_RESTART) + while ((ret = __hs_delete_key_from_ts_int(session, btree_id, key, ts, reinsert)) == WT_RESTART) WT_STAT_CONN_INCR(session, cache_hs_insert_restart); return (ret); @@ -980,7 +901,7 @@ __hs_fixup_out_of_order_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, hs_cursor, &tw.durable_stop_ts, &tw.durable_start_ts, &hs_upd_type, &hs_value)); /* Reinsert entry with earlier timestamp. */ - while ((ret = __hs_insert_record_with_btree_int(session, insert_cursor, btree, key, + while ((ret = __hs_insert_record_with_btree_int(session, insert_cursor, btree->id, key, (uint8_t)hs_upd_type, &hs_value, &start_time_point, &stop_time_point, *counter)) == WT_RESTART) ; @@ -1009,26 +930,64 @@ err: /* * __hs_delete_key_from_pos -- - * Delete an entire key's worth of data in the history store assuming that the input cursor is - * positioned at the beginning of the key range. + * Delete an entire key's worth of data in the history store. If we chose to reinsert the values + * the reinserted values will have 0 start and stop timestamps to ensure that they only use + * txnid based visibility rules. */ static int -__hs_delete_key_from_pos( - WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, uint32_t btree_id, const WT_ITEM *key) +__hs_delete_key_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, uint32_t btree_id, + const WT_ITEM *key, bool reinsert) { + WT_CURSOR *insert_cursor; WT_CURSOR_BTREE *hs_cbt; WT_DECL_RET; - WT_ITEM hs_key; + WT_HS_TIME_POINT start_time_point, stop_time_point; + WT_ITEM hs_key, hs_value; WT_UPDATE *upd; - wt_timestamp_t hs_start_ts; - uint64_t hs_counter; + wt_timestamp_t durable_timestamp, hs_start_ts, hs_stop_durable_ts; + uint64_t hs_counter, hs_insert_counter, hs_upd_type; uint32_t hs_btree_id; int cmp; + const char *open_cursor_cfg[] = {WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL}; hs_cbt = (WT_CURSOR_BTREE *)hs_cursor; + hs_insert_counter = 0; + WT_CLEAR(hs_key); + WT_CLEAR(hs_value); upd = NULL; + insert_cursor = NULL; + + if (reinsert) { + /* + * Determine the starting value of our counter, i.e. highest counter value of the timestamp + * range for timestamp 0. We'll be inserting at timestamp 0 and don't want to overwrite a + * currently existing counter. + * + * The cursor will also be positioned at the start of the range that we wish to start + * inserting. + */ + WT_WITHOUT_DHANDLE(session, + ret = __wt_open_cursor(session, WT_HS_URI, NULL, open_cursor_cfg, &insert_cursor)); + WT_ERR(ret); + F_SET(insert_cursor, WT_CURSTD_IGNORE_TOMBSTONE); + WT_ERR_NOTFOUND_OK( + __wt_hs_cursor_position(session, insert_cursor, btree_id, key, WT_TS_NONE, NULL), true); + + if (ret == WT_NOTFOUND) { + hs_insert_counter = 0; + ret = 0; + } else { + WT_ERR(insert_cursor->get_key( + insert_cursor, &hs_btree_id, &hs_key, &hs_start_ts, &hs_insert_counter)); + /* + * Increment the hs counter that we'll be using to insert with to avoid overwriting the + * record we just found. + */ + hs_insert_counter++; + } + } - /* If there is nothing else in history store, we're done here. */ + /* Begin iterating over the range of entries we expect to replace. */ for (; ret == 0; ret = __wt_hs_cursor_next(session, hs_cursor)) { WT_ERR(hs_cursor->get_key(hs_cursor, &hs_btree_id, &hs_key, &hs_start_ts, &hs_counter)); /* @@ -1049,6 +1008,33 @@ __hs_delete_key_from_pos( WT_STAT_CONN_INCR(session, cursor_next_hs_tombstone); continue; } + + /* + * Once we reinsert the entry below, we're not allowed to fail otherwise we'll be leaving + * our history store an invalid state. Anything that can potentially fail, such as heap + * allocation of the tombstone that we'll be using to remove the old value, should be + * performed before reinsertion. + */ + WT_ERR(__wt_upd_alloc_tombstone(session, &upd, NULL)); + + if (reinsert) { + WT_ERR(hs_cursor->get_value( + hs_cursor, &hs_stop_durable_ts, &durable_timestamp, &hs_upd_type, &hs_value)); + + start_time_point.ts = start_time_point.durable_ts = WT_TS_NONE; + start_time_point.txnid = hs_cbt->upd_value->tw.start_txn; + + stop_time_point.ts = stop_time_point.durable_ts = WT_TS_NONE; + stop_time_point.txnid = hs_cbt->upd_value->tw.stop_txn; + + /* Reinsert entry with zero timestamp. */ + while ((ret = __hs_insert_record_with_btree_int(session, insert_cursor, btree_id, + &hs_key, (uint8_t)hs_upd_type, &hs_value, &start_time_point, &stop_time_point, + hs_insert_counter)) == WT_RESTART) + ; + hs_insert_counter++; + WT_ERR(ret); + } /* * Since we're using internal functions to modify the row structure, we need to manually set * the comparison to an exact match. @@ -1057,17 +1043,28 @@ __hs_delete_key_from_pos( /* * Append a globally visible tombstone to the update list. This will effectively make the * value invisible and the key itself will eventually get removed during reconciliation. + * + * If anything fails after this point and we're reinserting we need to panic as it will + * leave our history store in an unexpected state with duplicate entries. */ - WT_ERR(__wt_upd_alloc_tombstone(session, &upd, NULL)); upd->txnid = WT_TXN_NONE; upd->start_ts = upd->durable_ts = WT_TS_NONE; - WT_ERR(__wt_hs_modify(hs_cbt, upd)); + if ((ret = __wt_hs_modify(hs_cbt, upd)) != 0) { + if (reinsert) + WT_ERR_PANIC(session, WT_PANIC, + "Failed to insert tombstone, history store now " + " contains duplicate values."); + else + WT_ERR(ret); + } upd = NULL; WT_STAT_CONN_INCR(session, cache_hs_key_truncate); } if (ret == WT_NOTFOUND) - return (0); + ret = 0; err: __wt_free(session, upd); + if (insert_cursor != NULL) + insert_cursor->close(insert_cursor); return (ret); } diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index d1bd86214c5..ca789921b1d 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -1076,10 +1076,10 @@ struct __wt_update { volatile uint8_t prepare_state; /* prepare state */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_UPDATE_CLEARED_HS 0x01u /* Update that cleared the history store. */ -#define WT_UPDATE_DS 0x02u /* Update has been written to the data store. */ -#define WT_UPDATE_HS 0x04u /* Update has been written to history store. */ -#define WT_UPDATE_OBSOLETE 0x08u /* Update that is obsolete. */ +#define WT_UPDATE_BEHIND_MIXED_MODE 0x01u /* Update that older than a mixed mode update. */ +#define WT_UPDATE_CLEARED_HS 0x02u /* Update that cleared the history store. */ +#define WT_UPDATE_DS 0x04u /* Update has been written to the data store. */ +#define WT_UPDATE_HS 0x08u /* Update has been written to history store. */ #define WT_UPDATE_PREPARE_RESTORED_FROM_DS 0x10u /* Prepared update restored from data store. */ #define WT_UPDATE_RESTORED_FAST_TRUNCATE 0x20u /* Fast truncate instantiation */ #define WT_UPDATE_RESTORED_FROM_DS 0x40u /* Update restored from data store. */ diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 3520b3b3941..ca28d17d4c5 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -458,10 +458,11 @@ struct __wt_connection_impl { uint32_t debug_log_cnt; /* Log file retention count */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CONN_DEBUG_CKPT_RETAIN 0x1u -#define WT_CONN_DEBUG_CURSOR_COPY 0x2u -#define WT_CONN_DEBUG_REALLOC_EXACT 0x4u -#define WT_CONN_DEBUG_SLOW_CKPT 0x8u +#define WT_CONN_DEBUG_CKPT_RETAIN 0x01u +#define WT_CONN_DEBUG_CORRUPTION_ABORT 0x02u +#define WT_CONN_DEBUG_CURSOR_COPY 0x04u +#define WT_CONN_DEBUG_REALLOC_EXACT 0x08u +#define WT_CONN_DEBUG_SLOW_CKPT 0x10u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t debug_flags; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index e0c268aacf8..044c32d7e98 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -736,6 +736,8 @@ extern int __wt_history_store_verify_one(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_hs_cursor_cache(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_cursor_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_cursor_next(WT_SESSION_IMPL *session, WT_CURSOR *cursor) @@ -749,8 +751,9 @@ extern int __wt_hs_cursor_prev(WT_SESSION_IMPL *session, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_cursor_search_near(WT_SESSION_IMPL *session, WT_CURSOR *cursor, int *exactp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_hs_delete_key_from_ts(WT_SESSION_IMPL *session, uint32_t btree_id, - const WT_ITEM *key, wt_timestamp_t ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_hs_delete_key_from_ts( + WT_SESSION_IMPL *session, uint32_t btree_id, const WT_ITEM *key, wt_timestamp_t ts, bool reinsert) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_format, uint64_t recno, WT_UPDATE_VALUE *upd_value, bool allow_prepare, WT_ITEM *on_disk_buf, WT_TIME_WINDOW *on_disk_tw) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 8b02e97779c..eaff9f721c4 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -399,7 +399,6 @@ struct __wt_connection_stats { int64_t cache_hs_read_squash; int64_t cache_hs_key_truncate_rts_unstable; int64_t cache_hs_key_truncate_rts; - int64_t cache_hs_key_truncate_mix_ts_restart; int64_t cache_hs_key_truncate; int64_t cache_hs_key_truncate_onpage_removal; int64_t cache_hs_key_truncate_mix_ts; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 24fbbeb1f24..7d4aed9df66 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1923,36 +1923,39 @@ struct __wt_connection { * checkpoint_retention, adjust log archiving to retain the log records of this number of * checkpoints. Zero or one means perform normal archiving., an integer between 0 and 1024; * default \c 0.} - * @config{ cursor_copy, if true\, use the system - * allocator to make a copy of any data returned by a cursor operation and return the copy - * instead. The copy is freed on the next cursor operation. This allows memory sanitizers - * to detect inappropriate references to memory owned by cursors., a boolean flag; default - * \c false.} - * @config{ eviction, if true\, modify internal algorithms - * to change skew to force history store eviction to happen more aggressively. This - * includes but is not limited to not skewing newest\, not favoring leaf pages\, and - * modifying the eviction score mechanism., a boolean flag; default \c false.} - * @config{ log_retention, adjust log archiving to retain at least - * this number of log files\, ignored if set to 0. (Warning: this option can remove log - * files required for recovery if no checkpoints have yet been done and the number of log - * files exceeds the configured value. As WiredTiger cannot detect the difference between a - * system that has not yet checkpointed and one that will never checkpoint\, it might - * discard log files before any checkpoint is done.)., an integer between 0 and 1024; - * default \c 0.} - * @config{ realloc_exact, if true\, reallocation of - * memory will only provide the exact amount requested. This will help with spotting memory - * allocation issues more easily., a boolean flag; default \c false.} - * @config{ rollback_error, return a WT_ROLLBACK error from a - * transaction operation about every Nth operation to simulate a collision., an integer - * between 0 and 10M; default \c 0.} - * @config{ slow_checkpoint, if - * true\, slow down checkpoint creation by slowing down internal page processing., a boolean + * @config{ corruption_abort, if true\, dump the core + * in the diagnostic mode on encountering the data corruption., a boolean flag; default \c + * true.} + * @config{ cursor_copy, if true\, use the system allocator to + * make a copy of any data returned by a cursor operation and return the copy instead. The + * copy is freed on the next cursor operation. This allows memory sanitizers to detect + * inappropriate references to memory owned by cursors., a boolean flag; default \c false.} + * @config{ eviction, if true\, modify internal algorithms to change + * skew to force history store eviction to happen more aggressively. This includes but is + * not limited to not skewing newest\, not favoring leaf pages\, and modifying the eviction + * score mechanism., a boolean flag; default \c false.} + * @config{ + * log_retention, adjust log archiving to retain at least this number of log files\, ignored + * if set to 0. (Warning: this option can remove log files required for recovery if no + * checkpoints have yet been done and the number of log files exceeds the configured value. + * As WiredTiger cannot detect the difference between a system that has not yet checkpointed + * and one that will never checkpoint\, it might discard log files before any checkpoint is + * done.)., an integer between 0 and 1024; default \c 0.} + * @config{ + * realloc_exact, if true\, reallocation of memory will only provide the exact amount + * requested. This will help with spotting memory allocation issues more easily., a boolean * flag; default \c false.} - * @config{ table_logging, if true\, write - * transaction related information to the log for all operations\, even operations for - * tables with logging turned off. This setting introduces a log format change that may - * break older versions of WiredTiger. These operations are informational and skipped in - * recovery., a boolean flag; default \c false.} + * @config{ rollback_error, return a + * WT_ROLLBACK error from a transaction operation about every Nth operation to simulate a + * collision., an integer between 0 and 10M; default \c 0.} + * @config{ + * slow_checkpoint, if true\, slow down checkpoint creation by slowing down internal page + * processing., a boolean flag; default \c false.} + * @config{ + * table_logging, if true\, write transaction related information to the log for all + * operations\, even operations for tables with logging turned off. This setting introduces + * a log format change that may break older versions of WiredTiger. These operations are + * informational and skipped in recovery., a boolean flag; default \c false.} * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; default empty.} * @config{eviction = (, eviction configuration options., a set of related configuration @@ -2542,10 +2545,12 @@ struct __wt_connection { * checkpoint_retention, adjust log archiving to retain the log records of this number of * checkpoints. Zero or one means perform normal archiving., an integer between 0 and 1024; default * \c 0.} - * @config{ cursor_copy, if true\, use the system allocator to make a - * copy of any data returned by a cursor operation and return the copy instead. The copy is freed - * on the next cursor operation. This allows memory sanitizers to detect inappropriate references - * to memory owned by cursors., a boolean flag; default \c false.} + * @config{ corruption_abort, if true\, dump the core in the + * diagnostic mode on encountering the data corruption., a boolean flag; default \c true.} + * @config{ cursor_copy, if true\, use the system allocator to make a copy of + * any data returned by a cursor operation and return the copy instead. The copy is freed on the + * next cursor operation. This allows memory sanitizers to detect inappropriate references to + * memory owned by cursors., a boolean flag; default \c false.} * @config{ * eviction, if true\, modify internal algorithms to change skew to force history store eviction to * happen more aggressively. This includes but is not limited to not skewing newest\, not favoring @@ -4914,880 +4919,875 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * an update */ #define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1103 -/*! - * cache: history store table truncation due to mixed timestamps that - * returned restart - */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_MIX_TS_RESTART 1104 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1105 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1104 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1106 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1105 /*! * cache: history store table truncation to remove range of updates due * to mixed timestamps */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_MIX_TS 1107 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_MIX_TS 1106 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1108 +#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1107 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1109 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1108 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1110 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1109 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1111 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1110 /*! cache: internal pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1112 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_QUEUED 1111 /*! cache: internal pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1113 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_SEEN 1112 /*! cache: internal pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1114 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL_PAGES_ALREADY_QUEUED 1113 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1115 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1114 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1116 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1115 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1117 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1116 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1118 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1117 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1119 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1118 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1120 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1119 /*! cache: operations timed out waiting for space in cache */ -#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1121 +#define WT_STAT_CONN_CACHE_TIMED_OUT_OPS 1120 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1122 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1121 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1123 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1122 /*! cache: page written requiring history store records */ -#define WT_STAT_CONN_CACHE_WRITE_HS 1124 +#define WT_STAT_CONN_CACHE_WRITE_HS 1123 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1125 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1124 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1126 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1125 /*! cache: pages evicted in parallel with checkpoint */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_IN_PARALLEL_WITH_CHECKPOINT 1127 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_IN_PARALLEL_WITH_CHECKPOINT 1126 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1128 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1127 /*! cache: pages queued for eviction post lru sorting */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1129 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_POST_LRU 1128 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1130 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1129 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1131 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1130 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1132 +#define WT_STAT_CONN_CACHE_READ 1131 /*! cache: pages read into cache after truncate */ -#define WT_STAT_CONN_CACHE_READ_DELETED 1133 +#define WT_STAT_CONN_CACHE_READ_DELETED 1132 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1134 +#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1133 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1135 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1134 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1136 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1135 /*! cache: pages seen by eviction walk that are already queued */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1137 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_ALREADY_QUEUED 1136 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1138 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1137 /*! * cache: pages selected for eviction unable to be evicted as the parent * page has overflow items */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1139 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_PARENT_HAS_OVERFLOW_ITEMS 1138 /*! * cache: pages selected for eviction unable to be evicted because of * active children on an internal page */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1140 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_ACTIVE_CHILDREN_ON_AN_INTERNAL_PAGE 1139 /*! * cache: pages selected for eviction unable to be evicted because of * failure in reconciliation */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1141 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1140 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1142 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1141 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1143 +#define WT_STAT_CONN_CACHE_WRITE 1142 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1144 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1143 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1145 +#define WT_STAT_CONN_CACHE_OVERHEAD 1144 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1146 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1145 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1147 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1146 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1148 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1147 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1149 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1148 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1150 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1149 /*! capacity: background fsync file handles considered */ -#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1151 +#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1150 /*! capacity: background fsync file handles synced */ -#define WT_STAT_CONN_FSYNC_ALL_FH 1152 +#define WT_STAT_CONN_FSYNC_ALL_FH 1151 /*! capacity: background fsync time (msecs) */ -#define WT_STAT_CONN_FSYNC_ALL_TIME 1153 +#define WT_STAT_CONN_FSYNC_ALL_TIME 1152 /*! capacity: bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1154 +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1153 /*! capacity: bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1155 +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1154 /*! capacity: bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1156 +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1155 /*! capacity: bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1157 +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1156 /*! capacity: bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1158 +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1157 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1159 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1158 /*! capacity: time waiting due to total capacity (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1160 +#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1159 /*! capacity: time waiting during checkpoint (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1161 +#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1160 /*! capacity: time waiting during eviction (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1162 +#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1161 /*! capacity: time waiting during logging (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_LOG 1163 +#define WT_STAT_CONN_CAPACITY_TIME_LOG 1162 /*! capacity: time waiting during read (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_READ 1164 +#define WT_STAT_CONN_CAPACITY_TIME_READ 1163 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_CONN_CC_PAGES_EVICT 1165 +#define WT_STAT_CONN_CC_PAGES_EVICT 1164 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_CONN_CC_PAGES_REMOVED 1166 +#define WT_STAT_CONN_CC_PAGES_REMOVED 1165 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1167 +#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1166 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_CONN_CC_PAGES_VISITED 1168 +#define WT_STAT_CONN_CC_PAGES_VISITED 1167 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1169 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1168 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1170 +#define WT_STAT_CONN_COND_AUTO_WAIT 1169 /*! * connection: auto adjusting condition wait raced to update timeout and * skipped updating */ -#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1171 +#define WT_STAT_CONN_COND_AUTO_WAIT_SKIPPED 1170 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1172 +#define WT_STAT_CONN_TIME_TRAVEL 1171 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1173 +#define WT_STAT_CONN_FILE_OPEN 1172 /*! connection: hash bucket array size for data handles */ -#define WT_STAT_CONN_BUCKETS_DH 1174 +#define WT_STAT_CONN_BUCKETS_DH 1173 /*! connection: hash bucket array size general */ -#define WT_STAT_CONN_BUCKETS 1175 +#define WT_STAT_CONN_BUCKETS 1174 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1176 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1175 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1177 +#define WT_STAT_CONN_MEMORY_FREE 1176 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1178 +#define WT_STAT_CONN_MEMORY_GROW 1177 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1179 +#define WT_STAT_CONN_COND_WAIT 1178 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1180 +#define WT_STAT_CONN_RWLOCK_READ 1179 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1181 +#define WT_STAT_CONN_RWLOCK_WRITE 1180 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1182 +#define WT_STAT_CONN_FSYNC_IO 1181 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1183 +#define WT_STAT_CONN_READ_IO 1182 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1184 +#define WT_STAT_CONN_WRITE_IO 1183 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1185 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1184 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1186 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1185 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1187 +#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1186 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1188 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1187 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1189 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1188 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1190 +#define WT_STAT_CONN_CURSOR_CACHE 1189 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1191 +#define WT_STAT_CONN_CURSOR_CREATE 1190 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1192 +#define WT_STAT_CONN_CURSOR_INSERT 1191 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1193 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1192 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1194 +#define WT_STAT_CONN_CURSOR_MODIFY 1193 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1195 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1194 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1196 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1195 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1197 +#define WT_STAT_CONN_CURSOR_NEXT 1196 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1198 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1197 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone in rollback to stable */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE_RTS 1199 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE_RTS 1198 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1200 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1199 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1201 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1200 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1202 +#define WT_STAT_CONN_CURSOR_RESTART 1201 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1203 +#define WT_STAT_CONN_CURSOR_PREV 1202 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1204 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1203 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone in rollback to stable */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE_RTS 1205 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE_RTS 1204 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1206 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1205 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1207 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1206 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1208 +#define WT_STAT_CONN_CURSOR_REMOVE 1207 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1209 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1208 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1210 +#define WT_STAT_CONN_CURSOR_RESERVE 1209 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1211 +#define WT_STAT_CONN_CURSOR_RESET 1210 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1212 +#define WT_STAT_CONN_CURSOR_SEARCH 1211 /*! cursor: cursor search history store calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_HS 1213 +#define WT_STAT_CONN_CURSOR_SEARCH_HS 1212 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1214 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1213 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1215 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1214 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1216 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1215 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1217 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1216 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1218 +#define WT_STAT_CONN_CURSOR_SWEEP 1217 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1219 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1218 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1220 +#define WT_STAT_CONN_CURSOR_UPDATE 1219 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1221 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1220 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1222 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1221 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1223 +#define WT_STAT_CONN_CURSOR_REOPEN 1222 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1224 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1223 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1225 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1224 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1226 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1225 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1227 +#define WT_STAT_CONN_DH_SWEEP_REF 1226 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1228 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1227 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1229 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1228 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1230 +#define WT_STAT_CONN_DH_SWEEP_TOD 1229 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1231 +#define WT_STAT_CONN_DH_SWEEPS 1230 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1232 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1231 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1233 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1232 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1234 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1233 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1235 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1234 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1236 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1235 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1237 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1236 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1238 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1237 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1239 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1238 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1240 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1239 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1241 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1240 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1242 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1241 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1243 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1242 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1244 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1243 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1245 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1244 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1246 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1245 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1247 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1246 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1248 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1247 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1249 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1248 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1250 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1249 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1251 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1250 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1252 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1251 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1253 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1252 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1254 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1253 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1255 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1254 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1256 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1255 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1257 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1256 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1258 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1257 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1259 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1258 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1260 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1259 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1261 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1260 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1262 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1261 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1263 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1262 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1264 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1263 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1265 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1264 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1266 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1265 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1267 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1266 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1268 +#define WT_STAT_CONN_LOG_FLUSH 1267 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1269 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1268 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1270 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1269 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1271 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1270 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1272 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1271 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1273 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1272 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1274 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1273 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1275 +#define WT_STAT_CONN_LOG_SCANS 1274 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1276 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1275 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1277 +#define WT_STAT_CONN_LOG_WRITE_LSN 1276 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1278 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1277 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1279 +#define WT_STAT_CONN_LOG_SYNC 1278 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1280 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1279 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1281 +#define WT_STAT_CONN_LOG_SYNC_DIR 1280 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1282 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1281 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1283 +#define WT_STAT_CONN_LOG_WRITES 1282 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1284 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1283 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1285 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1284 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1286 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1285 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1287 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1286 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1288 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1287 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1289 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1288 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1290 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1289 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1291 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1290 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1292 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1291 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1293 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1292 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1294 +#define WT_STAT_CONN_LOG_SLOT_RACES 1293 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1295 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1294 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1296 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1295 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1297 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1296 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1298 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1297 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1299 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1298 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1300 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1299 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1301 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1300 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1302 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1301 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1303 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1302 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1304 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1303 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1305 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1304 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1306 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1305 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1307 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1306 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1308 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1307 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1309 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1308 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1310 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1309 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1311 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1310 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1312 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1311 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1313 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1312 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1314 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1313 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1315 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1314 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1316 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1315 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1317 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1316 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1318 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1317 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1319 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1318 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1320 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1319 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1321 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1320 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1322 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1321 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1323 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1322 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1324 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1323 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1325 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1324 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1326 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1325 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1327 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1326 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1328 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1327 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1329 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1328 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1330 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1329 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1331 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1330 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1332 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1331 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1333 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1332 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1334 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1333 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1335 +#define WT_STAT_CONN_REC_PAGES 1334 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1336 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1335 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1337 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1336 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1338 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1337 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1339 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1338 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1340 +#define WT_STAT_CONN_REC_PAGE_DELETE 1339 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1341 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1340 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1342 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1341 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1343 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1342 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1344 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1343 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1345 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1344 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1346 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1345 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1347 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1346 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1348 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1347 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1349 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1348 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1350 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1349 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1351 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1350 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1352 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1351 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1353 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1352 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1354 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1353 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1355 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1354 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1356 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1355 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1357 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1356 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1358 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1357 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1359 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1358 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1360 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1359 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1361 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1360 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1362 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1361 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1363 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1362 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1364 +#define WT_STAT_CONN_SESSION_OPEN 1363 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1365 +#define WT_STAT_CONN_SESSION_QUERY_TS 1364 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1366 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1365 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1367 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1366 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1368 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1367 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1369 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1368 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1370 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1369 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1371 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1370 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1372 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1371 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1373 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1372 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1374 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1373 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1375 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1374 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1376 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1375 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1377 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1376 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1378 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1377 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1379 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1378 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1380 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1379 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1381 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1380 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1382 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1381 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1383 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1382 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1384 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1383 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1385 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1384 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1386 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1385 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1387 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1386 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1388 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1387 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1389 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1388 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1390 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1389 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1391 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1390 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1392 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1391 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1393 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1392 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1394 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1393 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1395 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1394 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1396 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1395 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1397 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1396 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1398 +#define WT_STAT_CONN_PAGE_SLEEP 1397 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1399 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1398 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1400 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1399 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1401 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1400 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1402 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1401 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1403 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1402 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1404 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1403 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1405 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1404 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1406 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1405 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1407 +#define WT_STAT_CONN_TXN_PREPARE 1406 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1408 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1407 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1409 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1408 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1410 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1409 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1411 +#define WT_STAT_CONN_TXN_QUERY_TS 1410 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1412 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1411 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1413 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1412 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1414 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1413 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1415 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1414 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1416 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1415 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1417 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1416 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1418 +#define WT_STAT_CONN_TXN_RTS 1417 /*! * transaction: rollback to stable hs records with stop timestamps older * than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1419 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1418 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1420 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1419 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1421 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1420 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1422 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1421 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1423 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1422 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1424 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1423 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1425 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1424 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1426 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1425 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1427 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1426 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1428 +#define WT_STAT_CONN_TXN_SET_TS 1427 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1429 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1428 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1430 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1429 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1431 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1430 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1432 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1431 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1433 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1432 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1434 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1433 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1435 +#define WT_STAT_CONN_TXN_BEGIN 1434 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1436 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1435 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1437 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1436 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1438 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1437 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1439 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1438 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1440 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1439 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1441 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1440 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1442 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1441 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1443 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1442 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1444 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1443 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1445 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1444 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1446 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1445 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1447 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1446 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1448 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1447 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1449 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1448 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1450 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1449 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1451 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1450 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1452 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1451 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1453 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1452 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1454 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1453 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1455 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1454 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1456 +#define WT_STAT_CONN_TXN_CHECKPOINT 1455 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1457 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1456 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1458 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1457 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1458 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1459 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1461 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1460 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1462 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1461 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1463 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1462 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1464 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1463 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1465 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1464 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1466 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1465 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1467 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1466 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1468 +#define WT_STAT_CONN_TXN_SYNC 1467 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1469 +#define WT_STAT_CONN_TXN_COMMIT 1468 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1470 +#define WT_STAT_CONN_TXN_ROLLBACK 1469 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1471 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1470 /*! * @} diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index c658377e51f..cac21e793bc 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -919,7 +919,8 @@ __wt_rec_row_leaf( */ if (!F_ISSET(session, WT_SESSION_NO_DATA_HANDLES)) { WT_ERR(__wt_hs_cursor_open(session)); - WT_ERR(__wt_hs_delete_key_from_ts(session, btree->id, tmpkey, WT_TS_NONE)); + WT_ERR(__wt_hs_delete_key_from_ts( + session, btree->id, tmpkey, WT_TS_NONE, false)); WT_ERR(__wt_hs_cursor_close(session)); WT_STAT_CONN_INCR(session, cache_hs_key_truncate_onpage_removal); } diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 629990b8830..2655b698a10 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -339,8 +339,11 @@ __wt_panic_func(WT_SESSION_IMPL *session, int error, const char *func, int line, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((format(printf, 5, 6))) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { + WT_CONNECTION_IMPL *conn; va_list ap; + conn = S2C(session); + /* * Ignore error returns from underlying event handlers, we already have an error value to * return. @@ -355,7 +358,7 @@ __wt_panic_func(WT_SESSION_IMPL *session, int error, const char *func, int line, * * If the connection has already panicked, just return the error. */ - if (session != NULL && F_ISSET(S2C(session), WT_CONN_PANIC)) + if (session != NULL && F_ISSET(conn, WT_CONN_PANIC)) return (WT_PANIC); /* @@ -369,15 +372,20 @@ __wt_panic_func(WT_SESSION_IMPL *session, int error, const char *func, int line, __eventv(session, false, WT_PANIC, func, line, "the process must exit and restart", ap)); va_end(ap); -/* - * Confusing #ifdef structure because gcc/clang knows the abort call won't return, and Visual Studio - * doesn't. - */ #if defined(HAVE_DIAGNOSTIC) - __wt_abort(session); /* Drop core if testing. */ - /* NOTREACHED */ + /* + * In the diagnostic builds, we want to drop core in case of panics that are not due to data + * corruption. A core could be useful in debugging. + * + * In the case of corruption, we want to be able to test the application's capability to salvage + * by returning an error code. But we do not want to lose the ability to drop core if required. + * Hence in the diagnostic mode, the application can set the debug flag to choose between + * dropping a core and returning an error. + */ + if (!F_ISSET(conn, WT_CONN_DATA_CORRUPTION) || + FLD_ISSET(conn->debug_flags, WT_CONN_DEBUG_CORRUPTION_ABORT)) + __wt_abort(session); #endif -#if !defined(HAVE_DIAGNOSTIC) || defined(_WIN32) /* * !!! * This function MUST handle a NULL WT_SESSION_IMPL handle. @@ -385,7 +393,7 @@ __wt_panic_func(WT_SESSION_IMPL *session, int error, const char *func, int line, * Panic the connection; */ if (session != NULL) - F_SET(S2C(session), WT_CONN_PANIC); + F_SET(conn, WT_CONN_PANIC); /* * !!! @@ -394,7 +402,6 @@ __wt_panic_func(WT_SESSION_IMPL *session, int error, const char *func, int line, * Order shall return. */ return (WT_PANIC); -#endif } /* diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 146643130db..340bddba73c 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -948,7 +948,6 @@ static const char *const __stats_connection_desc[] = { "cache: history store table reads requiring squashed modifies", "cache: history store table truncation by rollback to stable to remove an unstable update", "cache: history store table truncation by rollback to stable to remove an update", - "cache: history store table truncation due to mixed timestamps that returned restart", "cache: history store table truncation to remove an update", "cache: history store table truncation to remove range of updates due to key being removed from " "the data page during reconciliation", @@ -1466,7 +1465,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_hs_read_squash = 0; stats->cache_hs_key_truncate_rts_unstable = 0; stats->cache_hs_key_truncate_rts = 0; - stats->cache_hs_key_truncate_mix_ts_restart = 0; stats->cache_hs_key_truncate = 0; stats->cache_hs_key_truncate_onpage_removal = 0; stats->cache_hs_key_truncate_mix_ts = 0; @@ -1963,8 +1961,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cache_hs_key_truncate_rts_unstable += WT_STAT_READ(from, cache_hs_key_truncate_rts_unstable); to->cache_hs_key_truncate_rts += WT_STAT_READ(from, cache_hs_key_truncate_rts); - to->cache_hs_key_truncate_mix_ts_restart += - WT_STAT_READ(from, cache_hs_key_truncate_mix_ts_restart); to->cache_hs_key_truncate += WT_STAT_READ(from, cache_hs_key_truncate); to->cache_hs_key_truncate_onpage_removal += WT_STAT_READ(from, cache_hs_key_truncate_onpage_removal); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 044a3b00f02..0b5b7da92c0 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -923,13 +923,9 @@ done: * Perform rollback to stable only when the following conditions met. * 1. The connection is not read-only. A read-only connection expects that there shouldn't be * any changes that need to be done on the database other than reading. - * 2. A valid recovery timestamp. The recovery timestamp is the stable timestamp retrieved - * from the metadata checkpoint information to indicate the stable timestamp when the - * checkpoint happened. Anything updates newer than this timestamp must rollback. - * 3. The history store file was found in the metadata. + * 2. The history store file was found in the metadata. */ - if (hs_exists && !F_ISSET(conn, WT_CONN_READONLY) && - conn->txn_global.recovery_timestamp != WT_TS_NONE) { + if (hs_exists && !F_ISSET(conn, WT_CONN_READONLY)) { /* Start the eviction threads for rollback to stable if not already started. */ if (!eviction_started) { WT_ERR(__wt_evict_create(session)); @@ -956,7 +952,10 @@ done: * stable. */ conn->txn_global.stable_timestamp = conn->txn_global.recovery_timestamp; - conn->txn_global.has_stable_timestamp = true; + conn->txn_global.has_stable_timestamp = false; + + if (conn->txn_global.recovery_timestamp != WT_TS_NONE) + conn->txn_global.has_stable_timestamp = true; __wt_verbose(session, WT_VERB_RTS, "Performing recovery rollback_to_stable with stable timestamp: %s and oldest timestamp: " diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 56904666a8a..75569b8b057 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -967,17 +967,9 @@ __rollback_to_stable_btree(WT_SESSION_IMPL *session, wt_timestamp_t rollback_tim static int __rollback_to_stable_check(WT_SESSION_IMPL *session) { - WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_TXN_GLOBAL *txn_global; bool txn_active; - conn = S2C(session); - txn_global = &conn->txn_global; - - if (!txn_global->has_stable_timestamp) - WT_RET_MSG(session, EINVAL, "rollback_to_stable requires a stable timestamp"); - /* * Help the user comply with the requirement that there are no concurrent operations. Protect * against spurious conflicts with the sweep server: we exclude it from running concurrent with @@ -1147,17 +1139,19 @@ static int __rollback_to_stable_btree_apply(WT_SESSION_IMPL *session) { WT_CONFIG ckptconf; - WT_CONFIG_ITEM cval, durableval, key; + WT_CONFIG_ITEM cval, value, key; WT_CURSOR *cursor; WT_DECL_RET; WT_TXN_GLOBAL *txn_global; wt_timestamp_t max_durable_ts, newest_start_durable_ts, newest_stop_durable_ts, rollback_timestamp; + size_t addr_size; char ts_string[2][WT_TS_INT_STRING_SIZE]; const char *config, *uri; bool durable_ts_found, prepared_updates; txn_global = &S2C(session)->txn_global; + addr_size = 0; /* * Copy the stable timestamp, otherwise we'd need to lock it each time it's accessed. Even @@ -1191,32 +1185,62 @@ __rollback_to_stable_btree_apply(WT_SESSION_IMPL *session) WT_ERR(__wt_config_getones(session, config, "checkpoint", &cval)); __wt_config_subinit(session, &ckptconf, &cval); for (; __wt_config_next(&ckptconf, &key, &cval) == 0;) { - ret = __wt_config_subgets(session, &cval, "newest_start_durable_ts", &durableval); + ret = __wt_config_subgets(session, &cval, "newest_start_durable_ts", &value); if (ret == 0) { newest_start_durable_ts = - WT_MAX(newest_start_durable_ts, (wt_timestamp_t)durableval.val); + WT_MAX(newest_start_durable_ts, (wt_timestamp_t)value.val); durable_ts_found = true; } WT_ERR_NOTFOUND_OK(ret, false); - ret = __wt_config_subgets(session, &cval, "newest_stop_durable_ts", &durableval); + ret = __wt_config_subgets(session, &cval, "newest_stop_durable_ts", &value); if (ret == 0) { - newest_stop_durable_ts = - WT_MAX(newest_stop_durable_ts, (wt_timestamp_t)durableval.val); + newest_stop_durable_ts = WT_MAX(newest_stop_durable_ts, (wt_timestamp_t)value.val); durable_ts_found = true; } WT_ERR_NOTFOUND_OK(ret, false); - ret = __wt_config_subgets(session, &cval, "prepare", &durableval); + ret = __wt_config_subgets(session, &cval, "prepare", &value); if (ret == 0) { - if (durableval.val) + if (value.val) prepared_updates = true; } WT_ERR_NOTFOUND_OK(ret, false); + ret = __wt_config_subgets(session, &cval, "addr", &value); + if (ret == 0) + addr_size = value.len; + WT_ERR_NOTFOUND_OK(ret, false); } max_durable_ts = WT_MAX(newest_start_durable_ts, newest_stop_durable_ts); + + /* + * The rollback to stable will skip the tables during recovery in the following conditions + * 1. Empty table + * 2. Table has timestamped updates without a stable timestamp. + */ + if (F_ISSET(S2C(session), WT_CONN_RECOVERING) && + (addr_size == 0 || + (txn_global->stable_timestamp == WT_TS_NONE && max_durable_ts != WT_TS_NONE))) { + __wt_verbose(session, WT_VERB_RTS, "Skip rollback to stable on file %s because %s", uri, + addr_size == 0 ? "its checkpoint address length is 0" : + "it has timestamped updates and the stable timestamp is 0"); + continue; + } + + /* Set this flag to return error instead of panic if file is corrupted. */ + F_SET(session, WT_SESSION_QUIET_CORRUPT_FILE); ret = __wt_session_get_dhandle(session, uri, NULL, NULL, 0); - /* Ignore performing rollback to stable on files that don't exist. */ - if (ret == ENOENT) + F_CLR(session, WT_SESSION_QUIET_CORRUPT_FILE); + + /* + * Ignore performing rollback to stable on files that does not exist or the files where + * corruption is detected. + */ + if ((ret == ENOENT) || + (ret == WT_ERROR && F_ISSET(S2C(session), WT_CONN_DATA_CORRUPTION))) { + __wt_verbose(session, WT_VERB_RTS, + "Ignore performing rollback to stable on %s because the file %s", uri, + ret == ENOENT ? "does not exist" : "is corrupted."); continue; + } WT_ERR(ret); /* diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c index ef51a9dd19c..a97d314b9d2 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c @@ -332,32 +332,26 @@ copy_database(const char *sfx) } /* - * wt_open_corrupt -- + * open_with_corruption -- * Call wiredtiger_open and expect a corruption error. */ -static void wt_open_corrupt(const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void -wt_open_corrupt(const char *sfx) +open_with_corruption(const char *sfx) { WT_CONNECTION *conn; WT_DECL_RET; char buf[1024]; - /* The child should not abort the test in the message handler. Set it here, don't inherit. */ + /* We should not abort the test in the message handler. Set it here, don't inherit. */ test_abort = false; if (sfx != NULL) testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%s", home, sfx)); else testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home)); - /* - * Opening the database may cause a panic and core dump. Change dir to database directory so the - * core will be left someplace we will clean up. - */ - if (chdir(buf) != 0) - testutil_die(errno, "Child chdir: %s", home); + /* Don't abort in the diagnostic builds on detecting corruption. */ + ret = wiredtiger_open(buf, &event_handler, "debug_mode=(corruption_abort=false)", &conn); - ret = wiredtiger_open(NULL, &event_handler, NULL, &conn); /* * Not all out of sync combinations lead to corruption. We keep the previous checkpoint in the * file so some combinations of future or old turtle files and metadata files will succeed. @@ -365,31 +359,9 @@ wt_open_corrupt(const char *sfx) if (ret != WT_TRY_SALVAGE && ret != 0) fprintf(stderr, "OPEN_CORRUPT: wiredtiger_open returned %d\n", ret); testutil_assert(ret == WT_TRY_SALVAGE || ret == 0); - exit(EXIT_SUCCESS); -} -static int -open_with_error(const char *sfx) -{ - pid_t pid; - int status; - - /* - * Call wiredtiger_open. We expect to see a corruption panic so we run this in a forked process. - * In diagnostic mode, the panic will cause an abort and core dump. So we want to catch that and - * continue running with salvage. - */ - printf("=== open corrupt in child ===\n"); - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); - if (pid == 0) { /* child */ - wt_open_corrupt(sfx); - return (EXIT_SUCCESS); - } - /* parent */ - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); - return (EXIT_SUCCESS); + if (ret == 0) + testutil_check(conn->close(conn, NULL)); } static void @@ -443,7 +415,7 @@ open_normal(const char *sfx, TABLE_INFO *table_data) static void run_all_verification(const char *sfx, TABLE_INFO *t) { - testutil_check(open_with_error(sfx)); + open_with_corruption(sfx); open_with_salvage(sfx, t); open_normal(sfx, t); } @@ -471,6 +443,10 @@ main(int argc, char *argv[]) WT_DECL_RET; char buf[1024]; + /* Bypass this test for ASAN builds */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_ASAN")) + return (EXIT_SUCCESS); + opts = &_opts; memset(opts, 0, sizeof(*opts)); testutil_check(testutil_parse_opts(argc, argv, opts)); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 2dd16239845..8a627e7fac5 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -55,7 +55,7 @@ functions: set -o verbose if [ "$OS" != "Windows_NT" ]; then sh reconf - ${configure_env_vars|CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-ggdb -fPIC"} \ + ${configure_env_vars|CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-ggdb -fPIC"} PYTHON="python3" \ ../configure ${configure_python_setting|} \ ${posix_configure_flags|--enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL} fi @@ -68,8 +68,11 @@ functions: set -o errexit set -o verbose if [ "Windows_NT" == "$OS" ]; then - pip install scons==3.1.1 - scons-3.1.1.bat ${win_configure_flags|--enable-python=c:\\swigwin-3.0.2\\swig.exe --enable-diagnostic} ${smp_command|} + export "PATH=/cygdrive/c/Python39:/cygdrive/c/Python39/Scripts:$PATH" + + python --version + python -m pip install scons==3.1.1 + scons-3.1.1.bat "LIBPATH=c:\\python\\Python39\\libs" --enable-python=c:\\swigwin-3.0.2\\swig.exe --enable-diagnostic ${scons_smp_command|} else cd build_posix ${make_command|make} ${smp_command|} 2>&1 @@ -153,7 +156,11 @@ functions: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../test/suite/run.py ${unit_test_args|-v 2} ${smp_command|} 2>&1 + if [ "Windows_NT" = "$OS" ]; then + export "PATH=/cygdrive/c/Python39:/cygdrive/c/Python39/Scripts:$PATH" + export "PYTHONPATH=$(pwd)/../lang/python/wiredtiger):$(cygpath -w $(pwd)/../lang/python)" + fi + ${test_env_vars|} ${python_binary|python3} ../test/suite/run.py ${unit_test_args|-v 2} ${smp_command|} 2>&1 "format test": command: shell.exec params: @@ -587,7 +594,7 @@ tasks: - func: "make check all" vars: smp_command: -j 1 - test_env_vars: ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_SLOW_MACHINE=1 + test_env_vars: ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer TESTUTIL_SLOW_MACHINE=1 TESTUTIL_BYPASS_ASAN=1 - name: make-check-linux-no-ftruncate-test depends_on: @@ -1546,7 +1553,7 @@ tasks: set -o errexit set -o verbose - ${test_env_vars|} ${python_binary|python} ../../test/wtperf/test_conf_dump.py 2>&1 + ${test_env_vars|} ${python_binary|python3} ../../test/wtperf/test_conf_dump.py 2>&1 - name: compile-windows-alt tags: ["pull_request", "windows_only"] @@ -1561,8 +1568,10 @@ tasks: set -o errexit set -o verbose - pip install scons==3.1.1 - scons-3.1.1.bat ${smp_command|} "CFLAGS=/Gv /wd4090 /wd4996 /we4047 /we4024 /TC /we4100 /we4133" wiredtiger.dll libwiredtiger.lib + export "PATH=/cygdrive/c/Python39:/cygdrive/c/Python39/Scripts:$PATH" + python --version + python -m pip install scons==3.1.1 + scons-3.1.1.bat ${scons_smp_command|} "CFLAGS=/Gv /wd4090 /wd4996 /we4047 /we4024 /TC /we4100 /we4133" wiredtiger.dll libwiredtiger.lib - name: scons-check tags: ["pull_request", "windows_only"] @@ -1577,8 +1586,11 @@ tasks: set -o errexit set -o verbose - pip install scons==3.1.1 - scons-3.1.1.bat ${smp_command|} check + export "PATH=/cygdrive/c/Python39:/cygdrive/c/Python39/Scripts:$PATH" + python --version + python -m pip install scons==3.1.1 + # The check target is not run in parallel. + scons-3.1.1.bat check - name: fops tags: ["pull_request"] @@ -1863,7 +1875,7 @@ tasks: script: | set -o errexit set -o verbose - ${python_binary|python} syscall.py --verbose --preserve + ${python_binary|python3} syscall.py --verbose --preserve - name: checkpoint-filetypes-test commands: @@ -2225,7 +2237,7 @@ tasks: script: | set -o errexit set -o verbose - for i in {1..10}; do ${python_binary|python} split_stress.py; done + for i in {1..10}; do ${test_env_vars|} ${python_binary|python3} split_stress.py; done # The task name is ppc-zseries because this task will be used in both buildVariants - name: format-stress-ppc-zseries-test @@ -2381,6 +2393,7 @@ buildvariants: test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(echo "`grep -c ^processor /proc/cpuinfo` * 2" | bc) posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make tasks: - name: ".pull_request !.windows_only !.pull_request_compilers" @@ -2410,27 +2423,12 @@ buildvariants: - ubuntu1804-wt-build expansions: posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-lz4 --enable-snappy --enable-zlib --enable-zstd --enable-python + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make tasks: - name: ".pull_request_compilers" -- name: ubuntu1804-python3 - display_name: "! Ubuntu 18.04 (Python3)" - run_on: - - ubuntu1804-test - expansions: - test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) - smp_command: -j $(grep -c ^processor /proc/cpuinfo) - configure_python_setting: PYTHON=python3 - posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL - make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make - python_binary: python3 - tasks: - - name: compile - - name: ".unit_test" - - name: conf-dump-test - - name: ubuntu1804-stress-tests display_name: Ubuntu 18.04 Stress tests run_on: @@ -2444,6 +2442,7 @@ buildvariants: posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' tasks: - name: ".stress-test-1" - name: ".stress-test-2" @@ -2467,6 +2466,7 @@ buildvariants: test_env_vars: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' tasks: - name: compile-linux-no-ftruncate - name: make-check-linux-no-ftruncate-test @@ -2481,6 +2481,7 @@ buildvariants: test_env_vars: LD_PRELOAD=/usr/local/lib/libeatmydata.so PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) smp_command: -j $(grep -c ^processor /proc/cpuinfo) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' tasks: - name: compile - name: make-check-test @@ -2545,6 +2546,9 @@ buildvariants: display_name: "! Windows 64-bit" run_on: - windows-64-vs2017-test + expansions: + python_binary: 'python' + scons_smp_command: -j $(echo "$(getconf NPROCESSORS_ONLN 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1) * 2" | bc) tasks: - name: compile - name: ".windows_only" @@ -2558,6 +2562,7 @@ buildvariants: expansions: configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="-ggdb -fPIC" posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(sysctl -n hw.logicalcpu) make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future make test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH DYLD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) @@ -2573,6 +2578,7 @@ buildvariants: - ubuntu1804-test batchtime: 10080 # 7 days expansions: + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd) tasks: @@ -2589,6 +2595,7 @@ buildvariants: - ubuntu1804-zseries-build batchtime: 10080 # 7 days expansions: + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.lib top_srcdir=$(pwd)/.. top_builddir=$(pwd) tasks: @@ -2611,6 +2618,7 @@ buildvariants: posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' tasks: - name: compile - name: unit-test @@ -2631,6 +2639,7 @@ buildvariants: posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-snappy --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' tasks: - name: compile - name: unit-test diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 5cf39777fca..924cd61cc23 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -33,10 +33,9 @@ from __future__ import print_function import glob, json, os, random, re, sys -try: - xrange -except NameError: #python3 - xrange = range +if sys.version_info[0] <= 2: + print('WiredTiger requires Python version 3.0 or above') + sys.exit(1) # Set paths suitedir = sys.path[0] diff --git a/src/third_party/wiredtiger/test/suite/test_backup02.py b/src/third_party/wiredtiger/test/suite/test_backup02.py index c8886cc6e7b..e0f3fe1338f 100755 --- a/src/third_party/wiredtiger/test/suite/test_backup02.py +++ b/src/third_party/wiredtiger/test/suite/test_backup02.py @@ -26,11 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -try: - import Queue as queue # python2 -except ImportError: - import queue -import threading, time, wiredtiger, wttest +import queue, threading, time, wiredtiger, wttest from wtthread import backup_thread, checkpoint_thread, op_thread # test_backup02.py diff --git a/src/third_party/wiredtiger/test/suite/test_backup04.py b/src/third_party/wiredtiger/test/suite/test_backup04.py index 4ae7ee7766d..5de3b1647fc 100755 --- a/src/third_party/wiredtiger/test/suite/test_backup04.py +++ b/src/third_party/wiredtiger/test/suite/test_backup04.py @@ -33,10 +33,6 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios from wtthread import op_thread -try: - xrange -except NameError: #python3 - xrange = range # test_backup04.py # Utilities: wt backup @@ -70,14 +66,14 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): def populate(self, uri, dsize, rows): self.pr('populate: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) - for i in xrange(1, rows + 1): + for i in range(1, rows + 1): cursor[simple_key(cursor, i)] = str(i) + ':' + 'a' * dsize cursor.close() def update(self, uri, dsize, upd, rows): self.pr('update: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) - for i in xrange(1, rows + 1): + for i in range(1, rows + 1): cursor[simple_key(cursor, i)] = str(i) + ':' + upd * dsize cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_base05.py b/src/third_party/wiredtiger/test/suite/test_base05.py index 8d76ed96178..8e63715838c 100755 --- a/src/third_party/wiredtiger/test/suite/test_base05.py +++ b/src/third_party/wiredtiger/test/suite/test_base05.py @@ -28,10 +28,6 @@ import wiredtiger, wttest from wtscenario import make_scenarios -try: - xrange -except NameError: #python3 - xrange = range # test_base05.py # Cursor operations @@ -150,7 +146,7 @@ class test_base05(wttest.WiredTigerTestCase): """ nstrings = 2 << (n % 10) result = '' - for i in xrange(nstrings): + for i in range(nstrings): if (n + i) % 20 == 0: reflist = self.non_english_strings else: @@ -168,7 +164,7 @@ class test_base05(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor('table:' + self.table_name1) numbers = {} - for i in xrange(0, self.nentries): + for i in range(0, self.nentries): numbers[i] = i key = self.mixed_string(i) value = self.mixed_string(i+1) @@ -209,14 +205,14 @@ class test_base05(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor('table:' + self.table_name1) strlist = self.non_english_strings - for i in xrange(0, len(strlist)): + for i in range(0, len(strlist)): if convert: key = val = str(strlist[i]) else: key = val = strlist[i] cursor[key] = val - for i in xrange(0, len(strlist)): + for i in range(0, len(strlist)): if convert: key = val = str(strlist[i]) else: diff --git a/src/third_party/wiredtiger/test/suite/test_bulk01.py b/src/third_party/wiredtiger/test/suite/test_bulk01.py index f904e0e5db1..db5331eb603 100755 --- a/src/third_party/wiredtiger/test/suite/test_bulk01.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk01.py @@ -33,10 +33,6 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios -try: - xrange -except NameError: #python3 - xrange = range # Smoke test bulk-load. class test_bulk_load(wttest.WiredTigerTestCase): @@ -64,7 +60,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in xrange(1, 1000): + for i in range(1, 1000): cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Test a bulk-load triggers variable-length column-store RLE correctly. @@ -79,7 +75,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in xrange(1, 1000): + for i in range(1, 1000): cursor[simple_key(cursor, i)] = simple_value(cursor, i//7) # Test a bulk-load variable-length column-store append ignores any key. @@ -91,11 +87,11 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk,append") - for i in xrange(1, 1000): + for i in range(1, 1000): cursor[simple_key(cursor, 37)] = simple_value(cursor, i) cursor.close() cursor = self.session.open_cursor(uri, None, None) - for i in xrange(1, 1000): + for i in range(1, 1000): cursor.set_key(simple_key(cursor, i)) cursor.search() self.assertEqual(cursor.get_value(), simple_value(cursor, i)) @@ -109,7 +105,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in xrange(1, 1000): + for i in range(1, 1000): if i % 7 == 0: cursor[simple_key(cursor, i)] = simple_value(cursor, i) @@ -121,7 +117,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, None) # Verify all the records are there, in their proper state. - for i in xrange(1, 1000): + for i in range(1, 1000): cursor.set_key(simple_key(cursor, i)) if i % 7 == 0: cursor.search() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py index e40d72b5615..19fdb426eac 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py @@ -26,17 +26,9 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -try: - import Queue as queue # python2 -except ImportError: - import queue -import threading, time, wiredtiger, wttest +import queue, threading, time, wiredtiger, wttest from wtthread import checkpoint_thread, op_thread from wtscenario import make_scenarios -try: - xrange -except NameError: #python3 - xrange = range # test_checkpoint02.py # Run background checkpoints repeatedly while doing inserts and other @@ -60,7 +52,7 @@ class test_checkpoint02(wttest.WiredTigerTestCase): uris = list() uris.append(self.uri) my_data = 'a' * self.dsize - for i in xrange(self.nops): + for i in range(self.nops): if i % 191 == 0 and i != 0: work_queue.put_nowait(('b', i, my_data)) work_queue.put_nowait(('i', i, my_data)) diff --git a/src/third_party/wiredtiger/test/suite/test_config02.py b/src/third_party/wiredtiger/test/suite/test_config02.py index 3185ec54646..d704b6a2556 100755 --- a/src/third_party/wiredtiger/test/suite/test_config02.py +++ b/src/third_party/wiredtiger/test/suite/test_config02.py @@ -74,6 +74,12 @@ class test_config02(wttest.WiredTigerTestCase): WIREDTIGER_HOME is set to homeenv, if it is not null. configextra are any extra configuration strings needed on the open. """ + try: + os.putenv('SOMEVAR', 'somevalue') + os.unsetenv('SOMEVAR') + except: + self.skipTest('putenv and/or unsetenv not support on this OS') + return configarg = 'create' if configextra != None: configarg += ',' + configextra diff --git a/src/third_party/wiredtiger/test/suite/test_cursor12.py b/src/third_party/wiredtiger/test/suite/test_cursor12.py index a45ca4db9ce..79114828d2d 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor12.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor12.py @@ -180,15 +180,13 @@ class test_cursor12(wttest.WiredTigerTestCase): } ] - def setUp(self): - if sys.version_info[0] >= 3 and self.valuefmt == 'u': - # Python3 distinguishes bytes from strings - self.nullbyte = b'\x00' - self.spacebyte = b' ' + def nulls_to_spaces(self, bytes_or_str): + if self.valuefmt == 'u': + # The value is binary + return bytes_or_str.replace(b'\x00', b' ') else: - self.nullbyte = '\x00' - self.spacebyte = ' ' - super(test_cursor12, self).setUp() + # The value is a string + return bytes_or_str.replace('\x00', ' ') # Convert a string to the correct type for the value. def make_value(self, s): @@ -242,7 +240,7 @@ class test_cursor12(wttest.WiredTigerTestCase): self.assertEquals(c.search(), 0) v = c.get_value() expect = self.make_value(i['f']) - self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect) + self.assertEquals(self.nulls_to_spaces(v), expect) if not single: row = row + 1 @@ -259,7 +257,7 @@ class test_cursor12(wttest.WiredTigerTestCase): self.assertEquals(c.search(), 0) v = c.get_value() expect = self.make_value(i['f']) - self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect) + self.assertEquals(self.nulls_to_spaces(v), expect) if not single: row = row + 1 diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py b/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py index ff23fcc6985..bae583c7048 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py @@ -188,7 +188,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): if high < 0: high = 0 diff = high - low - nextra = (self.ord_byte(md5[4]) % (diff + 1)) + low + nextra = (md5[4] % (diff + 1)) + low extra = sha224.hexdigest() while len(extra) < nextra: extra = extra + extra diff --git a/src/third_party/wiredtiger/test/suite/test_hs18.py b/src/third_party/wiredtiger/test/suite/test_hs18.py new file mode 100644 index 00000000000..d5086ee293f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs18.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import time, wiredtiger, wttest, unittest + +def timestamp_str(t): + return '%x' % t + +# test_hs18.py +# Test various older reader scenarios +class test_hs18(wttest.WiredTigerTestCase): + conn_config = 'cache_size=5MB,eviction=(threads_max=1)' + session_config = 'isolation=snapshot' + + def check_value(self, cursor, value): + cursor.set_key(str(0)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), value) + cursor.reset() + + def start_txn(self, sessions, cursors, values, i): + # Start a transaction that will see update 0. + sessions[i].begin_transaction() + self.check_value(cursors[i], values[i]) + + def test_base_scenario(self): + uri = 'table:test_base_scenario' + self.session.create(uri, 'key_format=S,value_format=S') + session2 = self.setUpSessionOpen(self.conn) + cursor = self.session.open_cursor(uri) + cursor2 = session2.open_cursor(uri) + + value0 = 'f' * 500 + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 + value5 = 'e' * 500 + + # Insert an update at timestamp 3 + self.session.begin_transaction() + cursor[str(0)] = value0 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Start a long running transaction which could see update 0. + session2.begin_transaction() + self.check_value(cursor2, value0) + + # Insert an update at timestamp 5 + self.session.begin_transaction() + cursor[str(0)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + + # Insert another update at timestamp 10 + self.session.begin_transaction() + cursor[str(0)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Insert a bunch of contents to fill the cache + for i in range(2000, 10000): + self.session.begin_transaction() + cursor[str(i)] = value3 + self.session.commit_transaction() + + # Commit an update without a timestamp on our original key + self.session.begin_transaction() + cursor[str(0)] = value4 + self.session.commit_transaction() + + # Commit an update with timestamp 15 + self.session.begin_transaction() + cursor[str(0)] = value5 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + + # Check our value is still correct. + self.check_value(cursor2, value0) + + # Insert a bunch of other contents to trigger eviction + for i in range(10001, 11000): + self.session.begin_transaction() + cursor[str(i)] = value3 + self.session.commit_transaction() + + # Check our value is still correct. + self.check_value(cursor2, value0) + + # Test that we don't get the wrong value if we read with a timestamp originally. + def test_read_timestamp_weirdness(self): + uri = 'table:test_hs18' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + session2 = self.setUpSessionOpen(self.conn) + cursor2 = session2.open_cursor(uri) + session3 = self.setUpSessionOpen(self.conn) + cursor3 = session3.open_cursor(uri) + + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 + value5 = 'e' * 500 + + # Insert an update at timestamp 3 + self.session.begin_transaction() + cursor[str(0)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Start a long running transaction which could see update 0. + session2.begin_transaction('read_timestamp=' + timestamp_str(5)) + # Check our value is still correct. + self.check_value(cursor2, value1) + + # Insert another update at timestamp 10 + self.session.begin_transaction() + cursor[str(0)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Start a long running transaction which could see update 0. + session3.begin_transaction('read_timestamp=' + timestamp_str(5)) + # Check our value is still correct. + self.check_value(cursor3, value1) + + # Insert a bunch of contents to fill the cache + for i in range(1000, 10000): + self.session.begin_transaction() + cursor[str(i)] = value3 + self.session.commit_transaction() + + # Commit an update without a timestamp on our original key + self.session.begin_transaction() + cursor[str(0)] = value4 + self.session.commit_transaction() + + # Commit an update with timestamp 15 + self.session.begin_transaction() + cursor[str(0)] = value5 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + + # Check our value is still correct. + self.check_value(cursor2, value1) + self.check_value(cursor3, value1) + + # Insert a bunch of other contents to trigger eviction + for i in range(10001, 20000): + self.session.begin_transaction() + cursor[str(i)] = value3 + self.session.commit_transaction() + + # Check our value is still correct. + self.check_value(cursor2, value1) + # Here our value will be wrong as we're reading with a timestamp, and can now see a newer value. + self.check_value(cursor3, value2) + + # Test that forces us to ignore tombstone in order to not remove the first non timestamped updated. + def test_ignore_tombstone(self): + uri = 'table:test_ignore_tombstone' + self.session.create(uri, 'key_format=S,value_format=S') + session2 = self.setUpSessionOpen(self.conn) + cursor = self.session.open_cursor(uri) + cursor2 = session2.open_cursor(uri) + value0 = 'A' * 500 + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 + + # Insert an update without a timestamp + self.session.begin_transaction() + cursor[str(0)] = value0 + self.session.commit_transaction() + + # Start a long running transaction which could see update 0. + session2.begin_transaction() + + # Check our value is still correct. + self.check_value(cursor2, value0) + + # Insert an update at timestamp 5 + self.session.begin_transaction() + cursor[str(0)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + + # Insert another update at timestamp 10 + self.session.begin_transaction() + cursor[str(0)] = value2 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Insert a bunch of other contents to trigger eviction + for i in range(1, 10000): + self.session.begin_transaction() + cursor[str(i)] = value3 + self.session.commit_transaction() + + # Check our value is still correct. + self.check_value(cursor2, value0) + + # Commit an update without a timestamp on our original key + self.session.begin_transaction() + cursor[str(0)] = value4 + self.session.commit_transaction() + + # Insert a bunch of other contents to trigger eviction + for i in range(10000, 11000): + self.session.begin_transaction() + cursor[str(i)] = value3 + self.session.commit_transaction() + + # Check our value is still correct. + self.check_value(cursor2, value0) + + # Test older readers for each of the updates moved to the history store. + def test_multiple_older_readers(self): + uri = 'table:test_multiple_older_readers' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + + # The ID of the session corresponds the value it should see. + sessions = [] + cursors = [] + values = [] + for i in range(0, 5): + sessions.append(self.setUpSessionOpen(self.conn)) + cursors.append(sessions[i].open_cursor(uri)) + values.append(str(i) * 10) + + value_junk = 'aaaaa' * 100 + + # Insert an update at timestamp 3 + self.session.begin_transaction() + cursor[str(0)] = values[0] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Start a transaction that will see update 0. + self.start_txn(sessions, cursors, values, 0) + + # Insert an update at timestamp 5 + self.session.begin_transaction() + cursor[str(0)] = values[1] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + + # Start a transaction that will see update 1. + self.start_txn(sessions, cursors, values, 1) + + # Insert another update at timestamp 10 + self.session.begin_transaction() + cursor[str(0)] = values[2] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Start a transaction that will see update 2. + self.start_txn(sessions, cursors, values, 2) + + # Insert a bunch of other contents to trigger eviction + for i in range(1000, 10000): + self.session.begin_transaction() + cursor[str(i)] = value_junk + self.session.commit_transaction() + + # Commit an update without a timestamp on our original key + self.session.begin_transaction() + cursor[str(0)] = values[3] + self.session.commit_transaction() + + # Start a transaction that will see update 3. + self.start_txn(sessions, cursors, values, 3) + + # Commit an update with timestamp 15 + self.session.begin_transaction() + cursor[str(0)] = values[4] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + + # Insert a bunch of other contents to trigger eviction + for i in range(10001, 20000): + self.session.begin_transaction() + cursor[str(i)] = value_junk + self.session.commit_transaction() + + # Validate all values are visible and correct. + for i in range(0, 3): + cursors[i].set_key(str(0)) + self.assertEqual(cursors[i].search(), 0) + self.assertEqual(cursors[i].get_value(), values[i]) + cursors[i].reset() + + def test_multiple_older_readers_with_multiple_mixed_mode(self): + uri = 'table:test_multiple_older_readers' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + + # The ID of the session corresponds the value it should see. + sessions = [] + cursors = [] + values = [] + for i in range(0, 9): + sessions.append(self.setUpSessionOpen(self.conn)) + cursors.append(sessions[i].open_cursor(uri)) + values.append(str(i) * 10) + + value_junk = 'aaaaa' * 100 + + # Insert an update at timestamp 3 + self.session.begin_transaction() + cursor[str(0)] = values[0] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Start a transaction that will see update 0. + self.start_txn(sessions, cursors, values, 0) + + # Insert an update at timestamp 5 + self.session.begin_transaction() + cursor[str(0)] = values[1] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + + # Start a transaction that will see update 1. + self.start_txn(sessions, cursors, values, 1) + + # Insert another update at timestamp 10 + self.session.begin_transaction() + cursor[str(0)] = values[2] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Start a transaction that will see update 2. + self.start_txn(sessions, cursors, values, 2) + + # Insert a bunch of other contents to trigger eviction + for i in range(1000, 10000): + self.session.begin_transaction() + cursor[str(i)] = value_junk + self.session.commit_transaction() + + # Commit an update without a timestamp on our original key + self.session.begin_transaction() + cursor[str(0)] = values[3] + self.session.commit_transaction() + + # Start a transaction that will see update 4. + self.start_txn(sessions, cursors, values, 3) + + # Commit an update with timestamp 5 + self.session.begin_transaction() + cursor[str(0)] = values[4] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + + # Start a transaction that will see update 4. + self.start_txn(sessions, cursors, values, 4) + + # Commit an update with timestamp 10 + self.session.begin_transaction() + cursor[str(0)] = values[5] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Start a transaction that will see update 5. + self.start_txn(sessions, cursors, values, 5) + + # Commit an update with timestamp 15 + self.session.begin_transaction() + cursor[str(0)] = values[6] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + + # Start a transaction that will see update 6. + self.start_txn(sessions, cursors, values, 6) + + # Insert a bunch of other contents to trigger eviction + for i in range(10001, 20000): + self.session.begin_transaction() + cursor[str(i)] = value_junk + self.session.commit_transaction() + + # Validate all values are visible and correct. + for i in range(0, 6): + cursors[i].set_key(str(0)) + self.assertEqual(cursors[i].search(), 0) + self.assertEqual(cursors[i].get_value(), values[i]) + cursors[i].reset() + + # Commit an update without a timestamp on our original key + self.session.begin_transaction() + cursor[str(0)] = values[7] + self.session.commit_transaction() + + # Start a transaction that will see update 7. + self.start_txn(sessions, cursors, values, 7) + + # Commit an update with timestamp 5 + self.session.begin_transaction() + cursor[str(0)] = values[8] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + + # Insert a bunch of other contents to trigger eviction + for i in range(10001, 20000): + self.session.begin_transaction() + cursor[str(i)] = values[3] + self.session.commit_transaction() + + # Validate all values are visible and correct. + for i in range(0, 7): + cursors[i].set_key(str(0)) + self.assertEqual(cursors[i].search(), 0) + self.assertEqual(cursors[i].get_value(), values[i]) + cursors[i].reset() + + def test_modifies(self): + uri = 'table:test_modifies' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + session_ts_reader = self.setUpSessionOpen(self.conn) + cursor_ts_reader = session_ts_reader.open_cursor(uri) + + # The ID of the session corresponds the value it should see. + sessions = [] + cursors = [] + values = [] + for i in range(0, 5): + sessions.append(self.setUpSessionOpen(self.conn)) + cursors.append(sessions[i].open_cursor(uri)) + + value_junk = 'aaaaa' * 100 + + values.append('f' * 10) + values.append('a' + values[0]) + values.append('b' + values[1]) + values.append('g' * 10) + values.append('e' + values[3]) + + # Insert an update at timestamp 3 + self.session.begin_transaction() + cursor[str(0)] = values[0] + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Start a long running transaction which could see update 0. + self.start_txn(sessions, cursors, values, 0) + + # Insert an update at timestamp 5 + self.session.begin_transaction() + cursor.set_key(str(0)) + cursor.modify([wiredtiger.Modify('a', 0, 0)]) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + + session_ts_reader.begin_transaction('read_timestamp=3') + self.check_value(cursor_ts_reader, values[0]) + + # Start a long running transaction which could see modify 0. + self.start_txn(sessions, cursors, values, 1) + + # Insert another modify at timestamp 10 + self.session.begin_transaction() + cursor.set_key(str(0)) + cursor.modify([wiredtiger.Modify('b', 0, 0)]) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + + # Start a long running transaction which could see modify 1. + self.start_txn(sessions, cursors, values, 2) + + # Insert a bunch of contents to fill the cache + for i in range(2000, 10000): + self.session.begin_transaction() + cursor[str(i)] = value_junk + self.session.commit_transaction() + + # Commit a modify without a timestamp on our original key + self.session.begin_transaction() + cursor[str(0)] = values[3] + self.session.commit_transaction() + + # Start a long running transaction which could see value 5. + self.start_txn(sessions, cursors, values, 3) + + # Commit a final modify. + self.session.begin_transaction() + cursor.set_key(str(0)) + cursor.modify([wiredtiger.Modify('e', 0, 0)]) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + + # Start a long running transaction which could see modify 2. + sessions[4].begin_transaction() + # Check our values are still correct. + for i in range(0, 5): + self.check_value(cursors[i], values[i]) + + # Insert a bunch of other contents to trigger eviction + for i in range(10001, 11000): + self.session.begin_transaction() + cursor[str(i)] = value_junk + self.session.commit_transaction() + + # Check our values are still correct. + for i in range(0, 5): + self.check_value(cursors[i], values[i]) + + # Check our ts reader sees the value ahead of it + self.check_value(cursor_ts_reader, values[1]) diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump01.py b/src/third_party/wiredtiger/test/suite/test_jsondump01.py index 97af4764622..5a42cb38245 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_jsondump01.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump01.py @@ -48,7 +48,7 @@ class FakeCursor: def __iter__(self): return self - def __next__(self): # ready for python3.x + def __next__(self): return self.next() def next(self): diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump02.py b/src/third_party/wiredtiger/test/suite/test_jsondump02.py index eec0e5e97fd..9ae8dead18c 100755 --- a/src/third_party/wiredtiger/test/suite/test_jsondump02.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump02.py @@ -30,17 +30,6 @@ import os, sys import wiredtiger, wttest from suite_subprocess import suite_subprocess -# In Python2, Unicode and strings are different types, -# and need to be converted. In Python3, there is no separate -# unicode type, unicode characters are just embedded as UTF-8 -# in strings. -_python3 = (sys.version_info >= (3, 0, 0)) -def encode(s): - if _python3: - return s - else: - return s.encode('utf-8') - # test_jsondump.py # Test dump output from json cursors. class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): @@ -103,6 +92,9 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): Create JSON cursors and test them directly, also test dump/load commands. """ + import platform + if platform.system() == 'Darwin': + self.skipTest('JSON cursor test for OSX not yet working on Python3') extra_params = ',allocation_size=512,' +\ 'internal_page_max=16384,leaf_page_max=131072' self.session.create(self.table_uri1, @@ -128,8 +120,8 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): self.set_kv(self.table_uri1, 'KEY001', '\'\"({[]})\"\'\\, etc. allowed') # \u03c0 is pi in Unicode, converted by Python to UTF-8: 0xcf 0x80. # Here's how UTF-8 might be used. - self.set_kv(self.table_uri1, 'KEY002', encode(u'\u03c0')) - self.set_kv(self.table_uri1, 'KEY003', encode(u'\u0abc')) + self.set_kv(self.table_uri1, 'KEY002', u'\u03c0') + self.set_kv(self.table_uri1, 'KEY003', u'\u0abc') self.set_kv2(self.table_uri2, 'KEY000', 123, 'str0') self.set_kv2(self.table_uri2, 'KEY001', 234, 'str1') self.set_kv(self.table_uri3, 1, b'\x01\x02\x03') @@ -342,7 +334,7 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): val = ord('X') v[j] = val - # In Python3, we cannot simply shove random bytes with values >= 0x80 + # As of Python3, we cannot simply shove random bytes with values >= 0x80 # into a string, as strings are unicode aware, so we test only up to 0x80. # Real Unicode strings are tested elsewhere. def bytes_to_str(self, barray): diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py index c5d3a9b7050..afb96a09043 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py @@ -165,8 +165,12 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # After simulating a crash, search for the keys inserted. txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=false' - # Search keys with timestamp 5, ignore_prepare=false and expect the cursor value to be commit_value. - self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) + if self.commit == True: + # Search keys with timestamp 5, ignore_prepare=false and expect the cursor search to return WT_NOTFOUND. + self.search_keys_timestamp_and_ignore(ds, txn_config, None) + else: + # Search keys with timestamp 5, ignore_prepare=false and expect the cursor value to be commit_value. + self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' # Search keys with timestamp 20, ignore_prepare=true and expect the cursor search to return WT_NOTFOUND. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py index b16ee76a1f2..f3c93509d63 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py @@ -76,10 +76,6 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): self, uri_2, 0, key_format="i", value_format="S", config='log=(enabled=false)') ds_2.populate() - # Pin oldest and stable to timestamp 1. - self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + - ',stable_timestamp=' + timestamp_str(1)) - valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 valuec = "ccccc" * 100 @@ -113,12 +109,6 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): self.large_updates(uri_2, valued, ds_2, nrows, 0) self.check(valued, uri_2, nrows, 0) - # Pin stable to timestamp 20 if prepare otherwise 10. - if self.prepare: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) - else: - self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10)) - # Checkpoint to ensure that all the data is flushed. if not self.in_memory: self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_salvage.py b/src/third_party/wiredtiger/test/suite/test_salvage.py index 40ff8998659..425c1b78012 100755 --- a/src/third_party/wiredtiger/test/suite/test_salvage.py +++ b/src/third_party/wiredtiger/test/suite/test_salvage.py @@ -112,10 +112,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): """ Return a single byte from a file opened in binary mode. """ - c = fp.read(1) - if self.is_python3(): - c = c[0] # In python3, the read returns bytes (an array). - return c + return fp.read(1)[0] def damage_inner(self, tablename, unique): """ diff --git a/src/third_party/wiredtiger/test/suite/test_schema08.py b/src/third_party/wiredtiger/test/suite/test_schema08.py index 91925ffc975..ceda29fdb30 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema08.py +++ b/src/third_party/wiredtiger/test/suite/test_schema08.py @@ -140,10 +140,13 @@ class test_schema08(wttest.WiredTigerTestCase, suite_subprocess): # Make an initial copy as well as a copy for each LSN we save. # Truncate the log to the appropriate offset as we make each copy. olddir = "." + errfile="errfile.txt" for lsn in self.lsns: newdir = self.backup_pfx + str(lsn) outfile = newdir + '.txt' - self.runWt(['-R', '-h', newdir, 'list', '-v'], outfilename=outfile) + self.runWt(['-R', '-h', newdir, 'list', '-v'], errfilename=errfile, outfilename=outfile) + if os.path.isfile(errfile) and os.path.getsize(errfile) > 0: + self.check_file_contains(errfile,'No such file or directory') # Test that creating and dropping tables does not write individual # log records. diff --git a/src/third_party/wiredtiger/test/suite/test_txn16.py b/src/third_party/wiredtiger/test/suite/test_txn16.py index 9e2829246e5..6d7620218db 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn16.py +++ b/src/third_party/wiredtiger/test/suite/test_txn16.py @@ -92,11 +92,10 @@ class test_txn16(wttest.WiredTigerTestCase, suite_subprocess): cur_logs = fnmatch.filter(os.listdir(homedir), "*gerLog*") scur = set(cur_logs) sorig = set(orig_logs) - # There should never be overlap with the log files that - # were there originally. Mostly this checks that after - # opening with logging disabled and then re-enabled, we - # don't see log file 1. - self.assertEqual(scur.isdisjoint(sorig), True) + # There can be overlap with the log files that were + # there originally. Because some pages are rolled back + # as part of RTS. + self.assertEqual(scur.isdisjoint(sorig), False) if loop > 1: # We should be creating the same log files each time. for l in cur_logs: diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 3d81e2e9e6f..0f4b5e32907 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -69,7 +69,8 @@ def copy_for_crash_restart(olddir, newdir): class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): base_config = 'log=(archive=false,enabled,file_max=100K),' + \ - 'transaction_sync=(enabled,method=none),cache_size=1GB' + 'transaction_sync=(enabled,method=none),cache_size=1GB,' + \ + 'debug_mode=(corruption_abort=false),' conn_config = base_config corruption_type = [ ('removal', dict(kind='removal', f=lambda fname: @@ -286,30 +287,24 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): self.corrupt_log(newdir) #self.show_logs(newdir, 'after corruption') salvage_config = self.base_config + ',salvage=true' - errfile = 'list.err' - outfile = 'list.out' expect_fail = self.expect_recovery_failure() - # In cases of corruption, we cannot always call wiredtiger_open - # directly, because there may be a panic, and abort() is called - # in diagnostic mode which terminates the Python interpreter. - # - # Running any wt command externally to Python allows - # us to observe the failure or success safely. - # Use -R to force recover=on, which is the default for - # wiredtiger_open, (wt utilities normally have recover=error) - self.runWt(['-h', newdir, '-C', self.base_config, '-R', 'list'], - errfilename=errfile, outfilename=outfile, failure=expect_fail, - closeconn=False) - if expect_fail: - self.check_file_contains(errfile, - 'WT_TRY_SALVAGE: database corruption detected') + with self.expectedStdoutPattern('Failed wiredtiger_open'): + errmsg = '/WT_TRY_SALVAGE: database corruption detected/' + if self.kind == 'removal': + errmsg = '/No such file or directory/' + if self.kind == 'truncate': + errmsg = '/failed to read 128 bytes at offset 0/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.reopen_conn(newdir, self.base_config), errmsg) else: - self.check_empty_file(errfile) if self.expect_warning_corruption(): - self.check_file_contains(outfile, '/log file .* corrupted/') - self.check_file_contains(outfile, self.uri) + with self.expectedStdoutPattern('log file .* corrupted'): + self.reopen_conn(newdir, self.base_config) + else: + self.reopen_conn(newdir, self.base_config) + self.close_conn() found_records = self.recovered_records() expect = [x for x in range(0, found_records)] @@ -347,14 +342,15 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): expect.extend([self.nrecords, self.nrecords + 1]) copy_for_crash_restart(newdir, newdir2) self.checks(expect) - self.reopen_conn(newdir) + self.reopen_conn(newdir, self.conn_config) self.checks(expect) self.reopen_conn(newdir2, self.conn_config) self.checks(expect) class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): base_config = 'log=(archive=false,enabled,file_max=100K),' + \ - 'transaction_sync=(enabled,method=none),cache_size=1GB' + 'transaction_sync=(enabled,method=none),cache_size=1GB,' + \ + 'debug_mode=(corruption_abort=false),' conn_config = base_config # The type of corruption to be applied @@ -392,6 +388,7 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): openable = [ "removal:WiredTiger.basecfg", "removal:WiredTiger.turtle", + "removal:WiredTiger", "truncate:WiredTiger", "truncate:WiredTiger.basecfg", "truncate-middle:WiredTiger", @@ -475,23 +472,37 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): key = self.kind + ':' + self.filename return key not in self.not_salvageable - def run_wt_and_check(self, dir, errfile, outfile, expect_fail): - self.runWt(['-h', dir, '-C', self.base_config, '-R', 'list'], - errfilename=errfile, outfilename=outfile, failure=expect_fail, - closeconn=False) - + def run_wt_and_check(self, dir, expect_fail): if expect_fail: - errmsg = 'WT_TRY_SALVAGE: database corruption detected' + errmsg = '/WT_TRY_SALVAGE: database corruption detected/' if self.filename == 'WiredTigerHS.wt': if self.kind == 'removal': - errmsg = 'hs_exists' + errmsg = '/hs_exists/' elif self.kind == 'truncate': - errmsg = 'file size=0, alloc size=4096' - self.check_file_contains_one_of(errfile, [errmsg]) + errmsg = '/file size=0, alloc size=4096/' + if self.filename == 'WiredTiger.basecfg': + if self.kind == 'garbage-begin' or self.kind == 'garbage-end': + errmsg = '/Bad!Bad!Bad!/' + if self.filename == 'WiredTiger.wt': + if self.kind == 'truncate': + errmsg = '/is smaller than allocation size; file size=0, alloc size=4096/' + if self.kind == 'removal': + errmsg = '/No such file or directory/' + with self.expectedStdoutPattern('Failed wiredtiger_open'): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.reopen_conn(dir, self.conn_config), errmsg) + else: + # On non-windows platforms, we capture the renaming of WiredTiger.wt file. + if os.name != 'nt' and self.filename == 'WiredTiger.turtle' and self.kind == 'removal': + with self.expectedStderrPattern('File exists'): + self.reopen_conn(dir, self.conn_config) + self.captureout.checkAdditionalPattern(self, + 'unexpected file WiredTiger.wt found, renamed to WiredTiger.wt.1') + else: + self.reopen_conn(dir, self.conn_config) + self.close_conn() def test_corrupt_meta(self): - errfile = 'list.err' - outfile = 'list.out' newdir = "RESTART" newdir2 = "RESTART2" expect = list(range(0, self.nrecords)) @@ -516,15 +527,7 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): self.corrupt_meta(newdir) copy_for_crash_restart(newdir, newdir2) - # In cases of corruption, we cannot always call wiredtiger_open - # directly, because there may be a panic, and abort() is called - # in diagnostic mode which terminates the Python interpreter. - # - # Running any wt command externally to Python allows - # us to observe the failure or success safely. - # Use -R to force recover=on, which is the default for - # wiredtiger_open, (wt utilities normally have recover=error) - self.run_wt_and_check(newdir, errfile, outfile, not self.is_openable()) + self.run_wt_and_check(newdir, not self.is_openable()) for salvagedir in [ newdir, newdir2 ]: # Removing the 'WiredTiger.turtle' file has weird behavior: diff --git a/src/third_party/wiredtiger/test/suite/test_txn22.py b/src/third_party/wiredtiger/test/suite/test_txn22.py index 31d569951a7..8cf10408ad4 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn22.py +++ b/src/third_party/wiredtiger/test/suite/test_txn22.py @@ -170,5 +170,9 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.reopen_conn(salvagedir, salvage_config), '/.*/') + # The test may output the following error message while opening a file that + # does not exist. Ignore that. + self.ignoreStderrPatternIfExists('No such file or directory') + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_unicode01.py b/src/third_party/wiredtiger/test/suite/test_unicode01.py index 84a05b829b9..a4aab874928 100755 --- a/src/third_party/wiredtiger/test/suite/test_unicode01.py +++ b/src/third_party/wiredtiger/test/suite/test_unicode01.py @@ -27,7 +27,6 @@ # OTHER DEALINGS IN THE SOFTWARE. import sys, wiredtiger, wtscenario, wttest -_python3 = (sys.version_info >= (3, 0, 0)) # test_unicode01.py # Make sure UTF8 config can be passed to WT_SESSION::create. @@ -36,16 +35,7 @@ class test_unicode01(wttest.WiredTigerTestCase): def test_unicode(self): # We use valid Unicode characters that are examples in # the Unicode standard. - metadata_unicode = u'app_metadata={"name" : "Employ\u222b\u67d2\ud4db"}' - - # In Python2, Unicode and strings are different types, - # and need to be converted. In Python3, there is no separate - # unicode type, unicode characters are just embedded as UTF-8 - # in strings. - if _python3: - metadata_string = metadata_unicode - else: - metadata_string = metadata_unicode.encode('utf-8') + metadata_string = u'app_metadata={"name" : "Employ\u222b\u67d2\ud4db"}' self.session.create('table:t', metadata_string) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_util01.py b/src/third_party/wiredtiger/test/suite/test_util01.py index 30a53453647..9dae07f3ca6 100755 --- a/src/third_party/wiredtiger/test/suite/test_util01.py +++ b/src/third_party/wiredtiger/test/suite/test_util01.py @@ -30,8 +30,6 @@ import string, os, sys, random from suite_subprocess import suite_subprocess import wiredtiger, wttest -_python3 = (sys.version_info >= (3, 0, 0)) - def timestamp_str(t): return '%x' % t @@ -100,19 +98,11 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): def get_value(self, i): return self.get_bytes(i, 1000) - if _python3: - def _ord(self, byte): - return byte - - def _byte_to_str(self, byte): - return chr(byte) - - else: - def _ord(self, byte): - return ord(byte) + def _ord(self, byte): + return byte - def _byte_to_str(self, byte): - return byte + def _byte_to_str(self, byte): + return chr(byte) def dumpstr(self, s, hexoutput): """ @@ -227,6 +217,9 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): self.dump(False, True, None, None) def test_dump_api(self): + import platform + if platform.system() == 'Darwin': + self.skipTest('dump API test for OSX not yet working on Python3') self.dump(True, False, None, None) def test_dump_api_hex(self): diff --git a/src/third_party/wiredtiger/test/suite/test_verify.py b/src/third_party/wiredtiger/test/suite/test_verify.py index 18465320451..8279f772e3c 100755 --- a/src/third_party/wiredtiger/test/suite/test_verify.py +++ b/src/third_party/wiredtiger/test/suite/test_verify.py @@ -29,10 +29,6 @@ import os, struct from suite_subprocess import suite_subprocess import wiredtiger, wttest -try: - xrange -except NameError: #python3 - xrange = range # test_verify.py # Utilities: wt verify @@ -46,7 +42,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): """ cursor = self.session.open_cursor('table:' + tablename, None, None) key = '' - for i in xrange(0, self.nentries): + for i in range(0, self.nentries): key += str(i) cursor[key] = key + key cursor.close() @@ -134,7 +130,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 75) as f: - for i in xrange(0, 4096): + for i in range(0, 4096): f.write(struct.pack('B', 0)) # open_and_position closed the session/connection, reopen them now. @@ -153,7 +149,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 75) as f: - for i in xrange(0, 4096): + for i in range(0, 4096): f.write(struct.pack('B', 0)) self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) @@ -168,7 +164,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 25) as f: - for i in xrange(0, 100): + for i in range(0, 100): f.write(b'\x01\xff\x80') self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) @@ -186,7 +182,9 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): f.truncate(0) self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) - self.check_non_empty_file("verifyerr.out") + # The test may output the following error message while opening a file that + # does not exist. Ignore that. + self.ignoreStderrPatternIfExists('No such file or directory') def test_verify_process_zero_length(self): """ @@ -199,7 +197,9 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): f.truncate(0) self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) - self.check_non_empty_file("verifyerr.out") + # The test may output the following error message while opening a file that + # does not exist. Ignore that. + self.ignoreStderrPatternIfExists('No such file or directory') if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py index ac53d973e06..9954b5f2b94 100755 --- a/src/third_party/wiredtiger/test/suite/wtdataset.py +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -27,9 +27,6 @@ # OTHER DEALINGS IN THE SOFTWARE. # import sys -_python3 = (sys.version_info >= (3, 0, 0)) -if _python3: - xrange = range class BaseDataSet(object): """ @@ -54,7 +51,7 @@ class BaseDataSet(object): def fill(self): c = self.testcase.session.open_cursor(self.uri, None) - for i in xrange(1, self.rows + 1): + for i in range(1, self.rows + 1): c[self.key(i)] = self.value(i) c.close() @@ -77,9 +74,9 @@ class BaseDataSet(object): def key_by_format(i, key_format): if key_format == 'i' or key_format == 'r': return i - elif _python3 and key_format == 'u': + elif key_format == 'u': return bytes(('%015d' % i).encode()) - elif key_format == 'S' or key_format == 'u': + elif key_format == 'S': return str('%015d' % i) else: raise AssertionError( @@ -90,9 +87,9 @@ class BaseDataSet(object): def value_by_format(i, value_format): if value_format == 'i' or value_format == 'r': return i - elif _python3 and value_format == 'u': + elif value_format == 'u': return bytes((str(i) + ': abcdefghijklmnopqrstuvwxyz').encode()) - elif value_format == 'S' or value_format == 'u': + elif value_format == 'S': return str(i) + ': abcdefghijklmnopqrstuvwxyz' elif value_format == '8t': value = ( @@ -180,7 +177,7 @@ class SimpleIndexDataSet(SimpleDataSet): # Check values in the index. idxcursor = self.testcase.session.open_cursor(self.indexname) - for i in xrange(1, self.rows + 1): + for i in range(1, self.rows + 1): k = self.key(i) v = self.value(i) ik = (v, k) # The index key is columns=(v,k). @@ -368,7 +365,7 @@ class ProjectionIndexDataSet(BaseDataSet): self.testcase.assertEqual(i, self.rows) def check_index_cursor(self, cursor): - for i in xrange(1, self.rows + 1): + for i in range(1, self.rows + 1): k = self.key(i) v = self.value(i) ik = (v[2], v[1]) # The index key is (v2,v2) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 265917e589b..0e7bb3ae6f9 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -46,8 +46,6 @@ def shortenWithEllipsis(s, maxlen): s = s[0:maxlen-3] + '...' return s -_python3 = (sys.version_info >= (3, 0, 0)) - class CapturedFd(object): """ CapturedFd encapsulates a file descriptor (e.g. 1 or 2) that is diverted @@ -529,6 +527,10 @@ class WiredTigerTestCase(unittest.TestCase): if self.captureout.hasUnexpectedOutput(self): self.captureout.checkAdditionalPattern(self, pat) + def ignoreStderrPatternIfExists(self, pat): + if self.captureerr.hasUnexpectedOutput(self): + self.captureerr.checkAdditionalPattern(self, pat) + def assertRaisesWithMessage(self, exceptionType, expr, message): """ Like TestCase.assertRaises(), but also checks to see @@ -693,25 +695,7 @@ class WiredTigerTestCase(unittest.TestCase): """ return a recno key """ - if _python3: - return i - else: - return long(i) - - def ord_byte(self, b): - """ - return the 'ord' of a single byte. - In Python2 a set of bytes is represented as a string, and a single - byte is a string of length one. In Python3, bytes are an array of - ints, so no explicit ord() call is needed. - """ - if _python3: - return b - else: - return ord(b) - - def is_python3(self): - return _python3 + return i # print directly to tty, useful for debugging def tty(self, message): diff --git a/src/third_party/wiredtiger/test/suite/wtthread.py b/src/third_party/wiredtiger/test/suite/wtthread.py index 5853ba62526..1930d021d4e 100755 --- a/src/third_party/wiredtiger/test/suite/wtthread.py +++ b/src/third_party/wiredtiger/test/suite/wtthread.py @@ -26,11 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -try: - import Queue as queue # python2 -except ImportError: - import queue -import os, shutil, sys, threading, time, wiredtiger, wttest +import os, queue, shutil, sys, threading, time, wiredtiger, wttest from helper import compare_tables class checkpoint_thread(threading.Thread): |