diff options
128 files changed, 2444 insertions, 2044 deletions
diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 70e01507719..431f42e9e9c 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -717,6 +717,17 @@ config_sanity(WTPERF *wtperf) opts = wtperf->opts; + /* + * When backup functionalities are enabled, wtperf needs to use copy functions to write backup + * files. Direct I/O is not compatible with programs interacting with OS system-level copy + * functions, thus direct I/O and read only cannot be enabled along with backup. + */ + if (opts->backup_interval != 0) + if (strstr(opts->conn_config, "direct_io") || opts->readonly) { + fprintf(stderr, "direct_io and readonly cannot be used when backup is configured.\n"); + return (EINVAL); + } + /* Various intervals should be less than the run-time. */ if (opts->run_time > 0 && ((opts->checkpoint_threads != 0 && opts->checkpoint_interval > opts->run_time) || diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u-backup.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u-backup.wtperf index fdf6a83e94b..cd9213a2c11 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u-backup.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u-backup.wtperf @@ -24,4 +24,4 @@ sample_interval=10 sample_rate=1 threads=((count=10,reads=1),(count=10,updates=1)) warmup=120 -backup_interval=300 +backup_interval=1800 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly-backup.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-backup.wtperf index ef170c10201..bbd2b059a65 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly-backup.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-backup.wtperf @@ -1,13 +1,12 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=8),mmap=false" +conn_config="cache_size=50M,eviction=(threads_max=8)" table_config="type=file" icount=10000000 report_interval=5 run_time=120 populate_threads=1 -readonly=true threads=((count=16,reads=1)) # Add throughput/latency monitoring max_latency=2000 sample_interval=5 -backup_interval=40 +backup_interval=30 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-backup.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-backup.wtperf new file mode 100644 index 00000000000..c9e962faf55 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-backup.wtperf @@ -0,0 +1,17 @@ +# wtperf options file: small btree multi-database configuration +# Original cache was 500MB. Shared cache is 500MB * database_count. +conn_config="shared_cache=(name=pool,size=2500MB,chunk=1M),log=(enabled=false)" +database_count=10 +table_config="leaf_page_max=4k,internal_page_max=16k,leaf_item_max=1433,internal_item_max=3100,type=file" +# Likewise, divide original icount by database_count. +icount=50000 +populate_threads=1 +random_range=100000000 +checkpoint_interval=20 +checkpoint_threads=1 +report_interval=5 +run_time=20 +threads=((count=2,reads=1),(count=2,inserts=1)) +value_sz=100 +warmup=10 +backup_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress-backup.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress-backup.wtperf deleted file mode 100644 index ec42ec20789..00000000000 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress-backup.wtperf +++ /dev/null @@ -1,18 +0,0 @@ -# wtperf options file: multi-database configuration attempting to -# trigger slow operations by overloading CPU and disk. -# References Jira WT-2131 -conn_config="cache_size=2GB,eviction=(threads_min=2,threads_max=8),log=(enabled=false),direct_io=(data,checkpoint),buffer_alignment=4096,checkpoint_sync=true,checkpoint=(wait=60)" -table_config="allocation_size=4k,prefix_compression=false,split_pct=75,leaf_page_max=4k,internal_page_max=16k,leaf_item_max=1433,internal_item_max=3100,type=file" -# Divide original icount by database_count. -database_count=5 -icount=50000 -populate_threads=1 -random_range=50000000 -report_interval=5 -run_time=3600 -threads=((count=1,inserts=1),(count=10,reads=1)) -value_sz=100 -max_latency=1000 -sample_interval=5 -sample_rate=1 -backup_interval=240 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree.wtperf index e316dda9e88..9bcb4705bab 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree.wtperf @@ -1,7 +1,6 @@ # wtperf options file: small btree multi-database configuration # Original cache was 500MB. Shared cache is 500MB * database_count. conn_config="shared_cache=(name=pool,size=2500MB,chunk=1M),log=(enabled=false)" -#conn_config="cache_size=250MB,log=(enabled=false)" database_count=10 table_config="leaf_page_max=4k,internal_page_max=16k,leaf_item_max=1433,internal_item_max=3100,type=file" # Likewise, divide original icount by database_count. diff --git a/src/third_party/wiredtiger/build_cmake/README.md b/src/third_party/wiredtiger/build_cmake/README.md index 370df3732c7..42842bbe524 100644 --- a/src/third_party/wiredtiger/build_cmake/README.md +++ b/src/third_party/wiredtiger/build_cmake/README.md @@ -1,13 +1,15 @@ # Building WiredTiger with CMake -> *CMake support for building wiredtiger is an active work-in-progress. As of this time CMake can **only** build the WiredTiger library for POSIX platforms (Linux & Darwin) on x86 hosts. We suggest you continue using the autoconf build until further support is added.* +> *CMake support for building wiredtiger is an active work-in-progress. As of this time CMake can **only** build the WiredTiger library for POSIX platforms (Linux & Darwin). We suggest you continue using the SCons buildsystem when compiling for Windows.* ### Build Dependencies To build with CMake we **require** the following dependencies: * `cmake` : Official CMake install instructions found here: https://cmake.org/install/ - * *WiredTiger supports CMake 3.11+* + * *WiredTiger supports CMake 3.10+* * `ninja` : Official ninja install instructions found here: https://ninja-build.org/ +* `python3-dev` : Consult your package management application for installation instructions. This is needed for building with Python support. +* `swig`: Consult your package management application for installation instructions. This is needed for building with Python support. We also strongly suggest the following dependencies are also installed (for improved build times): @@ -23,6 +25,7 @@ Alternatively you can use your system's package manager to install the dependenc sudo apt-get install cmake cmake-curses-gui sudo apt-get install ccache sudo apt-get install ninja-build +sudo apt-get install python3-dev swig ``` ###### Install commands for Mac (using HomeBrew) @@ -31,10 +34,10 @@ sudo apt-get install ninja-build brew install ninja brew install ccache brew install cmake +brew install python +brew install swig ``` - - ### Building the WiredTiger Library > *The below commands are written for Linux and Darwin hosts. Windows instructions coming soon!* @@ -80,6 +83,7 @@ There are a number of additional configuration options you can pass to the CMake * `-DHAVE_DIAGNOSTIC=1` : Enable WiredTiger diagnostics * `-DHAVE_ATTACH=1` : Enable to pause for debugger attach on failure * `-DENABLE_STRICT=1` : Compile with strict compiler warnings enabled +* `-DENABLE_PYTHON=1` : Compile the python API * `-DCMAKE_INSTALL_PREFIX=<path-to-install-directory>` : Path to install directory --- @@ -118,3 +122,26 @@ $ cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/gcc.cmake -G Ninja ../. $ cd build $ cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/clang.cmake -G Ninja ../. ``` + +### Running WiredTiger C Tests + +The WiredTiger CMake build makes available a suite of C/C++ based tests (separate from the Python testsuite). The run all the tests available, ensure you're in the build directory and execute: + +```bash +ctest -j$(nproc) +``` + +To run with verbose output: + +```bash +ctest -j$(nproc) -VV +``` + +To run a specfic test: + +```bash +# Note: -R specifies a regex, where any matching test will be run +ctest -R <test_name> -j$(nproc) +``` + +See `ctest --help` for a range of additional supported options. diff --git a/src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake new file mode 100644 index 00000000000..94447ac9ca8 --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake @@ -0,0 +1,34 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# +include(CheckCCompilerFlag) + +set(WT_ARCH "aarch64" CACHE STRING "") +set(WT_OS "linux" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") + +# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions +# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based +# builds. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) + +# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. +set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") + +# ARMv8-A is the 64-bit ARM architecture, turn on the optional CRC instructions. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc" CACHE STRING "" FORCE) + +check_c_compiler_flag("-moutline-atomics" has_moutline_atomics) + +# moutline-atomics preserves backwards compatibility with Arm v8.0 systems but also supports +# using Arm v8.1 atomics. The latter can massively improve performance on larger Arm systems. +# The flag was back ported to gcc8, 9 and is the default in gcc10+. See if the compiler supports +# the flag. +if(has_moutline_atomics) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -moutline-atomics" CACHE STRING "" FORCE) +endif() +unset(has_moutline_atomics CACHE) diff --git a/src/third_party/wiredtiger/build_cmake/configs/auto.cmake b/src/third_party/wiredtiger/build_cmake/configs/auto.cmake index c3868bbdcd9..1df2578ed53 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/auto.cmake +++ b/src/third_party/wiredtiger/build_cmake/configs/auto.cmake @@ -282,6 +282,17 @@ config_compile( DEPENDS "HAVE_LIBPTHREAD" ) +include(TestBigEndian) +test_big_endian(is_big_endian) +if(NOT is_big_endian) + set(is_big_endian FALSE) +endif() +config_bool( + WORDS_BIGENDIAN + "If the target system is big endian" + DEFAULT ${is_big_endian} +) + set(wiredtiger_includes_decl) if(HAVE_SYS_TYPES_H) list(APPEND wiredtiger_includes_decl "#include <sys/types.h>") diff --git a/src/third_party/wiredtiger/build_cmake/configs/base.cmake b/src/third_party/wiredtiger/build_cmake/configs/base.cmake index 20ae0a319d0..a572fcb901e 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/base.cmake +++ b/src/third_party/wiredtiger/build_cmake/configs/base.cmake @@ -15,9 +15,9 @@ config_choice( "Target architecture for WiredTiger" OPTIONS "x86;WT_X86;" - "arm64;WT_ARM64;" - "ppc64;WT_PPC64;" - "zseries;WT_ZSERIES;" + "aarch64;WT_AARCH64;" + "ppc64le;WT_PPC64;" + "s390x;WT_S390X;" ) config_choice( @@ -166,7 +166,7 @@ config_string( config_string( VERSION_PATCH "Path version number for WiredTiger" - DEFAULT 0 + DEFAULT 1 ) @@ -174,11 +174,18 @@ string(TIMESTAMP config_date "%Y-%m-%d") config_string( VERSION_STRING "Version string for WiredTiger" - DEFAULT "\"WiredTiger 10.0.0 (${config_date})\"" + DEFAULT "\"WiredTiger ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH} (${config_date})\"" ) -if(HAVE_DIAGNOSTIC) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g" CACHE STRING "" FORCE) +if(HAVE_DIAGNOSTIC AND (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")) + # Avoid setting diagnostic flags if we are building with Debug mode. + # CMakes Debug config sets compilation with debug symbols by default. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g") endif() -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CC_OPTIMIZE_LEVEL}" CACHE STRING "" FORCE) +if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Release") + # Don't use the optimization level if we have specified a release config. + # CMakes Release config sets compilation to the highest optimization level + # by default. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CC_OPTIMIZE_LEVEL}") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/configs/modes.cmake b/src/third_party/wiredtiger/build_cmake/configs/modes.cmake new file mode 100644 index 00000000000..29b5672da6d --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/configs/modes.cmake @@ -0,0 +1,79 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +# Establishes build configuration modes we can use when compiling. + +# Create an ASAN build variant + +# Clang and GCC have slightly different linker names for the ASAN library. +set(libasan) +if("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") + set(libasan "-static-libsan") +else() + set(libasan "-static-libasan") +endif() + +set(CMAKE_C_FLAGS_ASAN + "${CMAKE_C_FLAGS_DEBUG} -fsanitize=address -fno-omit-frame-pointer" CACHE STRING + "Flags used by the C compiler for ASan build type or configuration." FORCE) + +set(CMAKE_CXX_FLAGS_ASAN + "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fno-omit-frame-pointer" CACHE STRING + "Flags used by the C++ compiler for ASan build type or configuration." FORCE) + +set(CMAKE_EXE_LINKER_FLAGS_ASAN + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=address ${libasan}" CACHE STRING + "Linker flags to be used to create executables for ASan build type." FORCE) + +set(CMAKE_SHARED_LINKER_FLAGS_ASAN + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=address ${libasan}" CACHE STRING + "Linker lags to be used to create shared libraries for ASan build type." FORCE) + +mark_as_advanced( + CMAKE_CXX_FLAGS_ASAN + CMAKE_C_FLAGS_ASAN + CMAKE_EXE_LINKER_FLAGS_ASAN + CMAKE_SHARED_LINKER_FLAGS_ASAN +) + +# Create an UBSAN build variant + +# Clang doesn't need to link ubsan, this is only a GCC requirement. +set(libubsan "") +if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") + set(libubsan "-lubsan") +endif() + +set(CMAKE_C_FLAGS_UBSAN + "${CMAKE_C_FLAGS_DEBUG} -fsanitize=undefined -fno-omit-frame-pointer" CACHE STRING + "Flags used by the C compiler for UBSan build type or configuration." FORCE) + +set(CMAKE_CXX_FLAGS_UBSAN + "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=undefined -fno-omit-frame-pointer" CACHE STRING + "Flags used by the C++ compiler for UBSan build type or configuration." FORCE) + +set(CMAKE_EXE_LINKER_FLAGS_UBSAN + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=undefined ${libubsan}" CACHE STRING + "Linker flags to be used to create executables for UBSan build type." FORCE) + +set(CMAKE_SHARED_LINKER_FLAGS_UBSAN + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=undefined ${libubsan}" CACHE STRING + "Linker lags to be used to create shared libraries for UBSan build type." FORCE) + +mark_as_advanced( + CMAKE_CXX_FLAGS_UBSAN + CMAKE_C_FLAGS_UBSAN + CMAKE_EXE_LINKER_FLAGS_UBSAN + CMAKE_SHARED_LINKER_FLAGS_UBSAN +) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "None" CACHE STRING "Choose the type of build, options are: None Debug Release ASan UBSan." FORCE) +endif() + +set(CMAKE_CONFIGURATION_TYPES None Debug Release ASan UBSan) diff --git a/src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake new file mode 100644 index 00000000000..4d1821c9e18 --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake @@ -0,0 +1,27 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +set(WT_ARCH "ppc64le" CACHE STRING "") +set(WT_OS "linux" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") + +# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions +# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based +# builds. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) + +# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. +set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") + +# Allow assembler to detect '.sx' file extensions. +list(APPEND CMAKE_ASM_SOURCE_FILE_EXTENSION "sx") + +# Our ASM-based checksum utility 'crc32.sx' triggers unused-macros diagnostic errors +# when compiling. To avoid editing the original source, override the usage '-Wunused-macros' +# for this specific file. +set_source_files_properties(src/checksum/power8/crc32.sx PROPERTIES COMPILE_FLAGS -Wno-unused-macros) diff --git a/src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake new file mode 100644 index 00000000000..639da9f297e --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake @@ -0,0 +1,22 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +set(WT_ARCH "s390x" CACHE STRING "") +set(WT_OS "linux" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") + +# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions +# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based +# builds. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) + +# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. +set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") + +# Allow assembler to detect '.sx' file extensions. +list(APPEND CMAKE_ASM_SOURCE_FILE_EXTENSION "sx") diff --git a/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in b/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in index f772594fc45..50937d6b95d 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in +++ b/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in @@ -129,6 +129,9 @@ /* Spinlock type from mutex.h. */ #cmakedefine SPINLOCK_TYPE @SPINLOCK_TYPE_CONFIG_VAR@ +/* Define to 1 if the target system is big endian */ +#cmakedefine WORDS_BIGENDIAN 1 + /* Version number of package */ #define VERSION "@VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_PATCH@" diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake index e7d5742fb27..4cd2d6882e4 100644 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake +++ b/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake @@ -13,7 +13,7 @@ set(WT_POSIX ON CACHE BOOL "") # Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions # e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based # builds. -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE") # Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") diff --git a/src/third_party/wiredtiger/build_cmake/helpers.cmake b/src/third_party/wiredtiger/build_cmake/helpers.cmake index 896518f92f5..d353c04aa8a 100644 --- a/src/third_party/wiredtiger/build_cmake/helpers.cmake +++ b/src/third_party/wiredtiger/build_cmake/helpers.cmake @@ -593,11 +593,11 @@ function(parse_filelist_source filelist output_var) # Determine architecture host for our filelist parse. if(WT_X86) set(arch_host "X86_HOST") - elseif(WT_ARM64) + elseif(WT_AARCH64) set(arch_host "ARM64_HOST") elseif(WT_PPC64) set(arch_host "POWERPC_HOST") - elseif(WT_ZSERIES) + elseif(WT_S390X) set(arch_host "ZSERIES_HOST") endif() # Determine platform host for our filelist parse. @@ -624,6 +624,13 @@ function(parse_filelist_source filelist output_var) list(GET file_contents 1 file_group) if ((${file_group} STREQUAL "${plat_host}") OR (${file_group} STREQUAL "${arch_host}")) list(APPEND output_files ${file_name}) + get_filename_component(file_ext ${file_name} EXT) + # POWERPC and ZSERIES hosts use the '.sx' extension for their ASM files. We need to + # manually tell CMake to ASM compile these files otherwise it will ignore them during + # compilation process. + if("${file_ext}" STREQUAL ".sx") + set_source_files_properties(${file_name} PROPERTIES LANGUAGE ASM) + endif() endif() else() message(FATAL_ERROR "filelist (${filelist}) has an unexpected format [Invalid Line: \"${file}]\"") diff --git a/src/third_party/wiredtiger/build_cmake/install/install.cmake b/src/third_party/wiredtiger/build_cmake/install/install.cmake index 8573b10ea1d..ec9b30234d6 100644 --- a/src/third_party/wiredtiger/build_cmake/install/install.cmake +++ b/src/third_party/wiredtiger/build_cmake/install/install.cmake @@ -35,32 +35,3 @@ if(WT_POSIX) DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig ) endif() - -# Install our wiredtiger compressor extensions (provided we have enabled/built them). -if(HAVE_BUILTIN_EXTENSION_LZ4 OR ENABLE_LZ4) - install(TARGETS wiredtiger_lz4 - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) -endif() - -if(HAVE_BUILTIN_EXTENSION_SNAPPY OR ENABLE_SNAPPY) - install(TARGETS wiredtiger_snappy - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) -endif() - -if(HAVE_BUILTIN_EXTENSION_ZLIB OR ENABLE_ZLIB) - install(TARGETS wiredtiger_zlib - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) -endif() - -if(HAVE_BUILTIN_EXTENSION_ZSTD OR ENABLE_ZSTD) - install(TARGETS wiredtiger_zstd - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) -endif() diff --git a/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake b/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake index 4ab3fa76ccb..b00dd1de6dd 100644 --- a/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake +++ b/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake @@ -6,34 +6,34 @@ # See the file LICENSE for redistribution information. # -string(APPEND clang_base_c_flags " -Weverything -Werror") -string(APPEND clang_base_c_flags " -Wno-cast-align") -string(APPEND clang_base_c_flags " -Wno-documentation-unknown-command") -string(APPEND clang_base_c_flags " -Wno-format-nonliteral") -string(APPEND clang_base_c_flags " -Wno-packed") -string(APPEND clang_base_c_flags " -Wno-padded") -string(APPEND clang_base_c_flags " -Wno-reserved-id-macro") -string(APPEND clang_base_c_flags " -Wno-zero-length-array") +list(APPEND clang_base_c_flags "-Weverything") +list(APPEND clang_base_c_flags "-Werror") +list(APPEND clang_base_c_flags "-Wno-cast-align") +list(APPEND clang_base_c_flags "-Wno-documentation-unknown-command") +list(APPEND clang_base_c_flags "-Wno-format-nonliteral") +list(APPEND clang_base_c_flags "-Wno-packed") +list(APPEND clang_base_c_flags "-Wno-padded") +list(APPEND clang_base_c_flags "-Wno-reserved-id-macro") +list(APPEND clang_base_c_flags "-Wno-zero-length-array") # We should turn on cast-qual, but not as a fatal error: see WT-2690. # For now, turn it off. -# string(APPEND clang_base_c_flags "-Wno-error=cast-qual") -string(APPEND clang_base_c_flags " -Wno-cast-qual") +list(APPEND clang_base_c_flags "-Wno-cast-qual") # Turn off clang thread-safety-analysis, it doesn't like some of the # code patterns in WiredTiger. -string(APPEND clang_base_c_flags " -Wno-thread-safety-analysis") +list(APPEND clang_base_c_flags "-Wno-thread-safety-analysis") # On Centos 7.3.1611, system header files aren't compatible with # -Wdisabled-macro-expansion. -string(APPEND clang_base_c_flags " -Wno-disabled-macro-expansion") +list(APPEND clang_base_c_flags "-Wno-disabled-macro-expansion") # We occasionally use an extra semicolon to indicate an empty loop or # conditional body. -string(APPEND clang_base_c_flags " -Wno-extra-semi-stmt") +list(APPEND clang_base_c_flags "-Wno-extra-semi-stmt") # Ignore unrecognized options. -string(APPEND clang_base_c_flags " -Wno-unknown-warning-option") +list(APPEND clang_base_c_flags "-Wno-unknown-warning-option") if(WT_DARWIN AND (CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.1)) # Apple clang has its own numbering system, and older OS X @@ -41,10 +41,17 @@ if(WT_DARWIN AND (CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.1)) # Apple's clang 4.1: # Apple clang version 4.1 # ((tags/Apple/clang-421.11.66) (based on LLVM 3.1svn) - string(APPEND clang_base_c_flags " -Wno-attributes") - string(APPEND clang_base_c_flags " -Wno-pedantic") - string(APPEND clang_base_c_flags " -Wno-unused-command-line-argument") + list(APPEND clang_base_c_flags "-Wno-attributes") + list(APPEND clang_base_c_flags "-Wno-pedantic") + list(APPEND clang_base_c_flags "-Wno-unused-command-line-argument") +endif() + +if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 10) + # Clang 10+ has added additional on-by-default diagnostics that isn't + # compatible with some of the code patterns in WiredTiger. + list(APPEND clang_base_c_flags "-Wno-implicit-fallthrough") + list(APPEND clang_base_c_flags "-Wno-implicit-int-float-conversion") endif() # Set our base clang flags to ensure it propogates to the rest of our build. -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${clang_base_c_flags}" CACHE STRING "" FORCE) +set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${clang_base_c_flags}" CACHE INTERNAL "" FORCE) diff --git a/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake b/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake index d417315fc4b..420a4457676 100644 --- a/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake +++ b/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake @@ -6,74 +6,74 @@ # See the file LICENSE for redistribution information. # -string(APPEND gcc_base_c_flags " -Wall") -string(APPEND gcc_base_c_flags " -Wextra") -string(APPEND gcc_base_c_flags " -Werror") -string(APPEND gcc_base_c_flags " -Waggregate-return") -string(APPEND gcc_base_c_flags " -Wbad-function-cast") -string(APPEND gcc_base_c_flags " -Wcast-align") -string(APPEND gcc_base_c_flags " -Wdeclaration-after-statement") -string(APPEND gcc_base_c_flags " -Wdouble-promotion") -string(APPEND gcc_base_c_flags " -Wfloat-equal") -string(APPEND gcc_base_c_flags " -Wformat-nonliteral") -string(APPEND gcc_base_c_flags " -Wformat-security") -string(APPEND gcc_base_c_flags " -Wformat=2") -string(APPEND gcc_base_c_flags " -Winit-self") -string(APPEND gcc_base_c_flags " -Wjump-misses-init") -string(APPEND gcc_base_c_flags " -Wmissing-declarations") -string(APPEND gcc_base_c_flags " -Wmissing-field-initializers") -string(APPEND gcc_base_c_flags " -Wmissing-prototypes") -string(APPEND gcc_base_c_flags " -Wnested-externs") -string(APPEND gcc_base_c_flags " -Wold-style-definition") -string(APPEND gcc_base_c_flags " -Wpacked") -string(APPEND gcc_base_c_flags " -Wpointer-arith") -string(APPEND gcc_base_c_flags " -Wpointer-sign") -string(APPEND gcc_base_c_flags " -Wredundant-decls") -string(APPEND gcc_base_c_flags " -Wshadow") -string(APPEND gcc_base_c_flags " -Wsign-conversion") -string(APPEND gcc_base_c_flags " -Wstrict-prototypes") -string(APPEND gcc_base_c_flags " -Wswitch-enum") -string(APPEND gcc_base_c_flags " -Wundef") -string(APPEND gcc_base_c_flags " -Wuninitialized") -string(APPEND gcc_base_c_flags " -Wunreachable-code") -string(APPEND gcc_base_c_flags " -Wunused") -string(APPEND gcc_base_c_flags " -Wwrite-strings") +list(APPEND gcc_base_c_flags "-Wall") +list(APPEND gcc_base_c_flags "-Wextra") +list(APPEND gcc_base_c_flags "-Werror") +list(APPEND gcc_base_c_flags "-Waggregate-return") +list(APPEND gcc_base_c_flags "-Wbad-function-cast") +list(APPEND gcc_base_c_flags "-Wcast-align") +list(APPEND gcc_base_c_flags "-Wdeclaration-after-statement") +list(APPEND gcc_base_c_flags "-Wdouble-promotion") +list(APPEND gcc_base_c_flags "-Wfloat-equal") +list(APPEND gcc_base_c_flags "-Wformat-nonliteral") +list(APPEND gcc_base_c_flags "-Wformat-security") +list(APPEND gcc_base_c_flags "-Wformat=2") +list(APPEND gcc_base_c_flags "-Winit-self") +list(APPEND gcc_base_c_flags "-Wjump-misses-init") +list(APPEND gcc_base_c_flags "-Wmissing-declarations") +list(APPEND gcc_base_c_flags "-Wmissing-field-initializers") +list(APPEND gcc_base_c_flags "-Wmissing-prototypes") +list(APPEND gcc_base_c_flags "-Wnested-externs") +list(APPEND gcc_base_c_flags "-Wold-style-definition") +list(APPEND gcc_base_c_flags "-Wpacked") +list(APPEND gcc_base_c_flags "-Wpointer-arith") +list(APPEND gcc_base_c_flags "-Wpointer-sign") +list(APPEND gcc_base_c_flags "-Wredundant-decls") +list(APPEND gcc_base_c_flags "-Wshadow") +list(APPEND gcc_base_c_flags "-Wsign-conversion") +list(APPEND gcc_base_c_flags "-Wstrict-prototypes") +list(APPEND gcc_base_c_flags "-Wswitch-enum") +list(APPEND gcc_base_c_flags "-Wundef") +list(APPEND gcc_base_c_flags "-Wuninitialized") +list(APPEND gcc_base_c_flags "-Wunreachable-code") +list(APPEND gcc_base_c_flags "-Wunused") +list(APPEND gcc_base_c_flags "-Wwrite-strings") # Non-fatal informational warnings. # We don't turn on the unsafe-loop-optimizations warning after gcc7, # it's too noisy to tolerate. Regardless, don't fail even when it's # configured. -string(APPEND gcc_base_c_flags " -Wno-error=unsafe-loop-optimizations") +list(APPEND gcc_base_c_flags "-Wno-error=unsafe-loop-optimizations") if(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 4.7) - string(APPEND gcc_base_c_flags " -Wno-c11-extensions") - string(APPEND gcc_base_c_flags " -Wunsafe-loop-optimizations") + list(APPEND gcc_base_c_flags "-Wno-c11-extensions") + list(APPEND gcc_base_c_flags "-Wunsafe-loop-optimizations") elseif(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 5) - string(APPEND gcc_base_c_flags " -Wunsafe-loop-optimizations") + list(APPEND gcc_base_c_flags "-Wunsafe-loop-optimizations") endif() if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 5) - string(APPEND gcc_base_c_flags " -Wformat-signedness") - string(APPEND gcc_base_c_flags " -Wjump-misses-init") - string(APPEND gcc_base_c_flags " -Wredundant-decls") - string(APPEND gcc_base_c_flags " -Wunused-macros") - string(APPEND gcc_base_c_flags " -Wvariadic-macros") + list(APPEND gcc_base_c_flags "-Wformat-signedness") + list(APPEND gcc_base_c_flags "-Wjump-misses-init") + list(APPEND gcc_base_c_flags "-Wredundant-decls") + list(APPEND gcc_base_c_flags "-Wunused-macros") + list(APPEND gcc_base_c_flags "-Wvariadic-macros") endif() if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 6) - string(APPEND gcc_base_c_flags " -Wduplicated-cond") - string(APPEND gcc_base_c_flags " -Wlogical-op") - string(APPEND gcc_base_c_flags " -Wunused-const-variable=2") + list(APPEND gcc_base_c_flags "-Wduplicated-cond") + list(APPEND gcc_base_c_flags "-Wlogical-op") + list(APPEND gcc_base_c_flags "-Wunused-const-variable=2") endif() if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 7) - string(APPEND gcc_base_c_flags " -Walloca") - string(APPEND gcc_base_c_flags " -Walloc-zero") - string(APPEND gcc_base_c_flags " -Wduplicated-branches") - string(APPEND gcc_base_c_flags " -Wformat-overflow=2") - string(APPEND gcc_base_c_flags " -Wformat-truncation=2") - string(APPEND gcc_base_c_flags " -Wrestrict") + list(APPEND gcc_base_c_flags "-Walloca") + list(APPEND gcc_base_c_flags "-Walloc-zero") + list(APPEND gcc_base_c_flags "-Wduplicated-branches") + list(APPEND gcc_base_c_flags "-Wformat-overflow=2") + list(APPEND gcc_base_c_flags "-Wformat-truncation=2") + list(APPEND gcc_base_c_flags "-Wrestrict") endif() if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 8) - string(APPEND gcc_base_c_flags " -Wmultistatement-macros") + list(APPEND gcc_base_c_flags "-Wmultistatement-macros") endif() # Set our base gcc flags to ensure it propogates to the rest of our build. -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${gcc_base_c_flags}" CACHE STRING "" FORCE) +set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${gcc_base_c_flags}" CACHE INTERNAL "" FORCE) diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake new file mode 100644 index 00000000000..98cc2a8892b --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake @@ -0,0 +1,15 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "aarch64-unknown-linux-gnu") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake new file mode 100644 index 00000000000..b82bbfe9b94 --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake @@ -0,0 +1,11 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PREFIX "aarch64-linux-gnu-") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake index 252f935976f..5b789f1134f 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake +++ b/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake @@ -6,7 +6,7 @@ # See the file LICENSE for redistribution information. # -cmake_minimum_required(VERSION 3.11.0) +cmake_minimum_required(VERSION 3.10.0) set(CMAKE_C_COMPILER "clang") set(CMAKE_C_COMPILER_ID "Clang") diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake index 25333335a2e..0fa64ed0037 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake +++ b/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake @@ -6,7 +6,7 @@ # See the file LICENSE for redistribution information. # -cmake_minimum_required(VERSION 3.11.0) +cmake_minimum_required(VERSION 3.10.0) if(NOT "${COMPILE_DEFINITIONS}" STREQUAL "") ### Additional check to overcome check_[symbol|include|function]_exits using toolchain file without passing WT_ARCH and WT_OS. diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake new file mode 100644 index 00000000000..f03d4fdbd97 --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake @@ -0,0 +1,15 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "powerpc64le-unknown-linux-gnu") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake new file mode 100644 index 00000000000..47e706c0d39 --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake @@ -0,0 +1,11 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PEFIX "powerpc64le-linux-gnu-") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake new file mode 100644 index 00000000000..f643b9c8df8 --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake @@ -0,0 +1,15 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "s390x-ibm-linux-gnu") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake new file mode 100644 index 00000000000..abfff342dfb --- /dev/null +++ b/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake @@ -0,0 +1,11 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PEFIX "s390x-linux-gnu-") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake index da6aa0762b2..e7130be256f 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake +++ b/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake @@ -6,9 +6,10 @@ # See the file LICENSE for redistribution information. # -set(TRIPLE_TARGET "x86_64-apple-darwin") - -set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) -set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") -set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") -set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "x86_64-apple-darwin") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake index 8ec2f561e54..7a463e46d96 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake +++ b/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake @@ -9,6 +9,6 @@ # We are not cross-compiling if our system is Darwin, hence the "x86_64-apple-darwin-" # prefix is not necessary when we are not cross-compiling. Just default to the host # installed 'gcc' binary. -if (NOT "${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Darwin") - set(CROSS_COMPILER_PREFIX "x86_64-apple-darwin-" CACHE INTERNAL "" FORCE) +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PREFIX "x86_64-apple-darwin-") endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake index 7eb4615b92d..126a3a1bbbe 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake +++ b/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake @@ -6,9 +6,10 @@ # See the file LICENSE for redistribution information. # -set(TRIPLE_TARGET "x86_64-pc-linux-gnu") - -set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) -set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") -set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") -set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "x86_64-pc-linux-gnu") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake index fc1f4307a08..2b36fb667d5 100644 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake +++ b/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake @@ -6,4 +6,6 @@ # See the file LICENSE for redistribution information. # -set(CROSS_COMPILER_PREFIX "x86_64-linux-gnu-" CACHE INTERNAL "" FORCE) +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PREFIX "x86_64-linux-gnu-") +endif() diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 6d65d9d217a..377fa72013a 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -279,9 +279,8 @@ file_config = format_meta + file_runtime_config + tiered_config + [ Config('block_allocation', 'best', r''' configure block allocation. Permitted values are \c "best" or \c "first"; the \c "best" configuration uses a best-fit algorithm, - the \c "first" configuration uses a first-available algorithm during block allocation, - the \c "log-structure" configuration allocates a new file for each checkpoint''', - choices=['best', 'first', 'log-structured',]), + the \c "first" configuration uses a first-available algorithm during block allocation''', + choices=['best', 'first',]), Config('allocation_size', '4KB', r''' the file unit allocation size, in bytes, must a power-of-two; smaller values decrease the file space required by overflow @@ -597,6 +596,9 @@ connection_runtime_config = [ versions of WiredTiger. These operations are informational and skipped in recovery.''', type='boolean'), + Config('update_restore_evict', 'false', r''' + if true, control all dirty page evictions through forcing update restore eviction.''', + type='boolean'), ]), Config('error_prefix', '', r''' prefix string for error messages'''), diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index f6bfd0f0fc7..5bdbe137fa2 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -211,8 +211,8 @@ src/support/thread_group.c src/support/timestamp.c src/support/update_vector.c src/tiered/tiered_config.c -src/tiered/tiered_cursor.c src/tiered/tiered_handle.c +src/tiered/tiered_work.c src/txn/txn.c src/txn/txn_ckpt.c src/txn/txn_ext.c diff --git a/src/third_party/wiredtiger/dist/s_function b/src/third_party/wiredtiger/dist/s_function index c9b4482d91b..d9014cf5394 100755 --- a/src/third_party/wiredtiger/dist/s_function +++ b/src/third_party/wiredtiger/dist/s_function @@ -40,6 +40,16 @@ done | python dist/s_function_loop.py | egrep '\{@[^@]*(WT_RET[_A-Z]*)\([^@]*(WT_ERR[_A-Z]*)\(.*err:' | sed -e 's/^\([^:]*\): *\([^:]*\):.*/\1:\2: mix of returns and jump to the error label within a loop/' +# Early exits from WT_WITH_* macros. +for f in $(grep -l 'WT_WITH_' $(find bench examples ext src test -name '*.c' -o -name '*_inline.h')); do + echo "===$f===" + cat $f +done | python dist/s_function_with.py 2>&1 $t || exit 1 +test -s $t && { + echo "Misuse of WT_WITH_* macros:" + cat $t +} + # Return of 0 in functions after a jump to the error label. for f in `find bench examples ext src test -name '*.c' -o -name '*._inline.h'`; do file_parse $f | diff --git a/src/third_party/wiredtiger/dist/s_function_with.py b/src/third_party/wiredtiger/dist/s_function_with.py new file mode 100644 index 00000000000..ff3438d2254 --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_function_with.py @@ -0,0 +1,49 @@ +# Examine WT_WITH_* macros and complain if an early exit out of the +# macro is done. These macros potentially restore state after they execute +# an expression, and an early exit, like a return/goto means the state +# will not be restored. +# +# Files appear verbatim, separated by a recognizable pattern. +import re, sys + +def get_indent(line): + return len(line) - len(line.lstrip()) + +filepat = re.compile('===(.*)===') +withpat = re.compile('WT_WITH_') +exitpat = re.compile('(WT_ERR|WT_RET|return\(|goto )') +filename = '' +linenum = 0 +with_indent = -1 +with_linenum = -1 +with_line = None +for line in sys.stdin: + m = filepat.search(line) + if m != None: + filename = m.group(1) + linenum = 0 + with_indent = -1 + with_linenum = -1 + with_line = None + linenum += 1 + + # Are we within a WT_WITH? + if with_indent >= 0: + cur_indent = get_indent(line) + # Have we "broken out" of the WT_WITH_.... macro? + # Our enforced formatting ensures this happens when the indent finishes. + if cur_indent <= with_indent: + # Yes, broken out + with_indent = -1 + with_linenum = -1 + with_line = None + elif exitpat.search(line): + print('{}: line {}: early exit: {}\n from: {}: line {}: {}'.format( + filename, linenum, line.strip(), filename, with_linenum, + with_line.strip())) + else: + m = withpat.search(line) + if m: + with_indent = get_indent(line) + with_linenum = linenum + with_line = line diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index db198204b10..e2603e2013d 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1450,6 +1450,7 @@ variadic vectorized versa vfprintf +vh vm vpack vpmsum diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 89a5578b362..e1bf0ebcc4e 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -865,6 +865,8 @@ conn_dsrc_stats = [ ########################################## StorageStat('tiered_object_size', 'tiered storage object size', 'no_clear,no_scale,size'), StorageStat('tiered_retention', 'tiered storage local retention time (secs)', 'no_clear,no_scale,size'), + StorageStat('tiered_work_units_created', 'tiered operations scheduled'), + StorageStat('tiered_work_units_dequeued', 'tiered operations dequeued and processed'), ########################################## # Transaction statistics diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 2c1e2219e30..69c05298149 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "bae0c1c914bc0fa92f3775c08650b65663094034" + "commit": "7374df6c344587d433853d01f0c6241428ab7a80" } diff --git a/src/third_party/wiredtiger/src/block/block_addr.c b/src/third_party/wiredtiger/src/block/block_addr.c index 15295bc02d3..f594d30e83e 100644 --- a/src/third_party/wiredtiger/src/block/block_addr.c +++ b/src/third_party/wiredtiger/src/block/block_addr.c @@ -14,12 +14,12 @@ * reference so it can be called repeatedly to load a buffer. */ static int -__block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *logidp, wt_off_t *offsetp, +__block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) { uint64_t l, o, s, c; - if (block->log_structured) + if (block->has_objects) WT_RET(__wt_vunpack_uint(pp, 0, &l)); else l = 0; @@ -38,9 +38,9 @@ __block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *logidp, wt */ if (s == 0) { *offsetp = 0; - *logidp = *sizep = *checksump = 0; + *objectidp = *sizep = *checksump = 0; } else { - *logidp = (uint32_t)l; + *objectidp = (uint32_t)l; *offsetp = (wt_off_t)(o + 1) * block->allocsize; *sizep = (uint32_t)s * block->allocsize; *checksump = (uint32_t)c; @@ -53,8 +53,8 @@ __block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *logidp, wt * Convert the filesystem components into its address cookie. */ int -__wt_block_addr_to_buffer( - WT_BLOCK *block, uint8_t **pp, uint32_t logid, wt_off_t offset, uint32_t size, uint32_t checksum) +__wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, wt_off_t offset, + uint32_t size, uint32_t checksum) { uint64_t l, o, s, c; @@ -63,12 +63,12 @@ __wt_block_addr_to_buffer( o = WT_BLOCK_INVALID_OFFSET; l = s = c = 0; } else { - l = logid; + l = objectid; o = (uint64_t)offset / block->allocsize - 1; s = size / block->allocsize; c = checksum; } - if (block->log_structured) + if (block->has_objects) WT_RET(__wt_vpack_uint(pp, 0, l)); WT_RET(__wt_vpack_uint(pp, 0, o)); WT_RET(__wt_vpack_uint(pp, 0, s)); @@ -82,10 +82,10 @@ __wt_block_addr_to_buffer( * reference. */ int -__wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *logidp, wt_off_t *offsetp, +__wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) { - return (__block_buffer_to_addr(block, &p, logidp, offsetp, sizep, checksump)); + return (__block_buffer_to_addr(block, &p, objectidp, offsetp, sizep, checksump)); } /* @@ -97,14 +97,14 @@ __wt_block_addr_invalid( WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) { wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(session); WT_UNUSED(addr_size); WT_UNUSED(live); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); #ifdef HAVE_DIAGNOSTIC /* @@ -116,7 +116,7 @@ __wt_block_addr_invalid( #endif /* Check if the address is past the end of the file. */ - return (logid == block->logid && offset + size > block->size ? EINVAL : 0); + return (objectid == block->objectid && offset + size > block->size ? EINVAL : 0); } /* @@ -128,16 +128,16 @@ __wt_block_addr_string( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) { wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(addr_size); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); /* Printable representation. */ WT_RET(__wt_buf_fmt(session, buf, - "[%" PRIu32 ": %" PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", logid, + "[%" PRIu32 ": %" PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", objectid, (uintmax_t)offset, (uintmax_t)offset + size, size, checksum)); return (0); @@ -160,12 +160,12 @@ __block_buffer_to_ckpt( pp = &p; WT_RET(__block_buffer_to_addr( - block, pp, &ci->root_logid, &ci->root_offset, &ci->root_size, &ci->root_checksum)); + block, pp, &ci->root_objectid, &ci->root_offset, &ci->root_size, &ci->root_checksum)); WT_RET(__block_buffer_to_addr( - block, pp, &ci->alloc.logid, &ci->alloc.offset, &ci->alloc.size, &ci->alloc.checksum)); + block, pp, &ci->alloc.objectid, &ci->alloc.offset, &ci->alloc.size, &ci->alloc.checksum)); WT_RET(__block_buffer_to_addr( - block, pp, &ci->avail.logid, &ci->avail.offset, &ci->avail.size, &ci->avail.checksum)); - WT_RET(__block_buffer_to_addr(block, pp, &ci->discard.logid, &ci->discard.offset, + block, pp, &ci->avail.objectid, &ci->avail.offset, &ci->avail.size, &ci->avail.checksum)); + WT_RET(__block_buffer_to_addr(block, pp, &ci->discard.objectid, &ci->discard.offset, &ci->discard.size, &ci->discard.checksum)); WT_RET(__wt_vunpack_uint(pp, 0, &a)); ci->file_size = (wt_off_t)a; @@ -209,9 +209,9 @@ __wt_block_ckpt_to_buffer( WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail) { uint64_t a; - uint32_t logid; + uint32_t objectid; - logid = block->logid; + objectid = block->objectid; if (ci->version != WT_BM_CHECKPOINT_VERSION) WT_RET_MSG(session, WT_ERROR, "unsupported checkpoint version"); @@ -220,16 +220,16 @@ __wt_block_ckpt_to_buffer( (*pp)++; WT_RET(__wt_block_addr_to_buffer( - block, pp, logid, ci->root_offset, ci->root_size, ci->root_checksum)); + block, pp, objectid, ci->root_offset, ci->root_size, ci->root_checksum)); WT_RET(__wt_block_addr_to_buffer( - block, pp, logid, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); + block, pp, objectid, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); if (skip_avail) WT_RET(__wt_block_addr_to_buffer(block, pp, 0, 0, 0, 0)); else WT_RET(__wt_block_addr_to_buffer( - block, pp, logid, ci->avail.offset, ci->avail.size, ci->avail.checksum)); + block, pp, objectid, ci->avail.offset, ci->avail.size, ci->avail.checksum)); WT_RET(__wt_block_addr_to_buffer( - block, pp, logid, ci->discard.offset, ci->discard.size, ci->discard.checksum)); + block, pp, objectid, ci->discard.offset, ci->discard.size, ci->discard.checksum)); a = (uint64_t)ci->file_size; WT_RET(__wt_vpack_uint(pp, 0, a)); a = ci->ckpt_size; diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 0f4af019b52..8194b8212de 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -95,7 +95,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint if (ci->root_offset != WT_BLOCK_INVALID_OFFSET) { endp = root_addr; WT_ERR(__wt_block_addr_to_buffer( - block, &endp, ci->root_logid, ci->root_offset, ci->root_size, ci->root_checksum)); + block, &endp, ci->root_objectid, ci->root_offset, ci->root_size, ci->root_checksum)); *root_addr_sizep = WT_PTRDIFF(endp, root_addr); WT_ERR(__wt_block_tiered_load(session, block, ci)); @@ -115,7 +115,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint * the end of the file, that was done when the checkpoint was first written (re-writing the * checkpoint might possibly make it relevant here, but it's unlikely enough I don't bother). */ - if (!checkpoint && !block->log_structured) + if (!checkpoint && !block->has_objects) WT_ERR(__wt_block_truncate(session, block, ci->file_size)); if (0) { @@ -239,9 +239,9 @@ __wt_block_checkpoint( */ if (buf == NULL) { ci->root_offset = WT_BLOCK_INVALID_OFFSET; - ci->root_logid = ci->root_size = ci->root_checksum = 0; + ci->root_objectid = ci->root_size = ci->root_checksum = 0; } else - WT_ERR(__wt_block_write_off(session, block, buf, &ci->root_logid, &ci->root_offset, + WT_ERR(__wt_block_write_off(session, block, buf, &ci->root_objectid, &ci->root_offset, &ci->root_size, &ci->root_checksum, data_checksum, true, false)); /* @@ -612,7 +612,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) * lists, and the freed blocks will then be included when writing the live extent lists. */ WT_CKPT_FOREACH (ckptbase, ckpt) { - if (F_ISSET(ckpt, WT_CKPT_FAKE) || !F_ISSET(ckpt, WT_CKPT_DELETE) || block->log_structured) + if (F_ISSET(ckpt, WT_CKPT_FAKE) || !F_ISSET(ckpt, WT_CKPT_DELETE) || block->has_objects) continue; if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) @@ -750,7 +750,7 @@ live_update: * TODO: tiered: for now we are switching files on a checkpoint, we'll want to do it only on * flush_tier. */ - if (block->log_structured) + if (block->has_objects) WT_ERR(__wt_block_tiered_newfile(session, block)); #ifdef HAVE_DIAGNOSTIC diff --git a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c index 0b2629cce83..8b87fb5ab62 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c @@ -220,15 +220,15 @@ __wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, char **met const WT_PAGE_HEADER *dsk; wt_off_t ext_off, ext_size, offset; uint64_t len, nblocks, write_gen; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; const uint8_t *p, *t; bool found; *metadatap = *checkpoint_listp = NULL; WT_RET(__wt_buf_init(session, checkpoint, WT_BLOCK_CHECKPOINT_BUFFER)); - /* TODO: scan all log IDs. */ - logid = 0; + /* TODO: tiered: scan all object IDs. */ + objectid = 0; /* * Initialize a pair of structures that track the best and current checkpoints found so far. @@ -282,7 +282,7 @@ __wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, char **met * block isn't valid, skip to the next possible block. */ if (__wt_block_offset_invalid(block, offset, size) || - __wt_block_read_off(session, block, tmp, logid, offset, size, checksum) != 0) { + __wt_block_read_off(session, block, tmp, objectid, offset, size, checksum) != 0) { size = WT_BTREE_MIN_ALLOC_SIZE; continue; } diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 740abcf6141..735b4d5c4f4 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -136,13 +136,13 @@ __wt_block_compact_page_skip( WT_EXT *ext; WT_EXTLIST *el; wt_off_t limit, offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(addr_size); *skipp = true; /* Return a default skip. */ /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); /* * If this block is in the chosen percentage of the file and there's a block on the available diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index 310330da831..2d95ed973cd 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -563,24 +563,24 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, { WT_DECL_RET; wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(addr_size); WT_STAT_DATA_INCR(session, block_free); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); - __wt_verbose(session, WT_VERB_BLOCK, "free %" PRIu32 ": %" PRIdMAX "/%" PRIdMAX, logid, + __wt_verbose(session, WT_VERB_BLOCK, "free %" PRIu32 ": %" PRIdMAX "/%" PRIdMAX, objectid, (intmax_t)offset, (intmax_t)size); #ifdef HAVE_DIAGNOSTIC WT_RET(__wt_block_misplaced(session, block, "free", offset, size, true, __func__, __LINE__)); #endif - if (logid == block->logid) { + if (objectid == block->objectid) { WT_RET(__wt_block_ext_prealloc(session, 5)); __wt_spin_lock(session, &block->live_lock); - ret = __wt_block_off_free(session, block, logid, offset, (wt_off_t)size); + ret = __wt_block_off_free(session, block, objectid, offset, (wt_off_t)size); __wt_spin_unlock(session, &block->live_lock); } else { /* TODO: update stats about older files to drive garbage collection. */ @@ -595,7 +595,7 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, */ int __wt_block_off_free( - WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, wt_off_t offset, wt_off_t size) + WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, wt_off_t offset, wt_off_t size) { WT_DECL_RET; @@ -603,7 +603,7 @@ __wt_block_off_free( WT_ASSERT(session, WT_SESSION_BTREE_SYNC_SAFE(session, S2BT(session))); /* TODO: track stats for old files to drive garbage collection. */ - if (logid != block->logid) + if (objectid != block->objectid) return (0); /* @@ -1103,7 +1103,8 @@ __wt_block_extlist_read( return (0); WT_RET(__wt_scr_alloc(session, el->size, &tmp)); - WT_ERR(__wt_block_read_off(session, block, tmp, el->logid, el->offset, el->size, el->checksum)); + WT_ERR( + __wt_block_read_off(session, block, tmp, el->objectid, el->offset, el->size, el->checksum)); p = WT_BLOCK_HEADER_BYTE(tmp->mem); WT_ERR(__wt_extlist_read_pair(&p, &off, &size)); @@ -1163,7 +1164,7 @@ __wt_block_extlist_write( WT_EXT *ext; WT_PAGE_HEADER *dsk; size_t size; - uint32_t logid, entries; + uint32_t objectid, entries; uint8_t *p; WT_RET(__block_extlist_dump(session, block, el, "write")); @@ -1221,8 +1222,8 @@ __wt_block_extlist_write( /* Write the extent list to disk. */ WT_ERR(__wt_block_write_off( - session, block, tmp, &logid, &el->offset, &el->size, &el->checksum, true, true, true)); - WT_UNUSED(logid); /* TODO check */ + session, block, tmp, &objectid, &el->offset, &el->size, &el->checksum, true, true, true)); + WT_UNUSED(objectid); /* TODO: tiered: check */ /* * Remove the allocated blocks from the system's allocation list, extent blocks never appear on diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 169c1cd5d87..c41de4aaaaa 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -99,10 +99,10 @@ __block_destroy(WT_SESSION_IMPL *session, WT_BLOCK *block) __wt_free(session, block->name); - if (block->log_structured && block->lfh != NULL) { - for (i = 0; i < block->max_logid; i++) - WT_TRET(__wt_close(session, &block->lfh[i])); - __wt_free(session, block->lfh); + if (block->has_objects && block->ofh != NULL) { + for (i = 0; i < block->max_objectid; i++) + WT_TRET(__wt_close(session, &block->ofh[i])); + __wt_free(session, block->ofh); } if (block->fh != NULL) @@ -182,7 +182,6 @@ __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[ WT_ERR(__wt_config_gets(session, cfg, "block_allocation", &cval)); block->allocfirst = WT_STRING_MATCH("first", cval.str, cval.len); - block->log_structured = WT_STRING_MATCH("log-structured", cval.str, cval.len); /* Configuration: optional OS buffer cache maximum size. */ WT_ERR(__wt_config_gets(session, cfg, "os_cache_max", &cval)); diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 80dbb3aac21..08c1ce8067f 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -21,7 +21,7 @@ __wt_bm_preload(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_FH *fh; WT_FILE_HANDLE *handle; wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; bool mapped; block = bm->block; @@ -29,9 +29,9 @@ __wt_bm_preload(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_STAT_CONN_INCR(session, block_preload); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); - WT_RET(__wt_block_fh(session, block, logid, &fh)); + WT_RET(__wt_block_fh(session, block, objectid, &fh)); handle = fh->handle; mapped = bm->map != NULL && offset + size <= (wt_off_t)bm->maplen; if (mapped && handle->fh_map_preload != NULL) @@ -64,19 +64,19 @@ __wt_bm_read( WT_FH *fh; WT_FILE_HANDLE *handle; wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; bool mapped; WT_UNUSED(addr_size); block = bm->block; /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); /* * Map the block if it's possible. */ - WT_RET(__wt_block_fh(session, block, logid, &fh)); + WT_RET(__wt_block_fh(session, block, objectid, &fh)); handle = fh->handle; mapped = bm->map != NULL && offset + size <= (wt_off_t)bm->maplen; if (mapped && handle->fh_map_preload != NULL) { @@ -100,7 +100,7 @@ __wt_bm_read( #endif /* Read the block. */ __wt_capacity_throttle(session, size, WT_THROTTLE_READ); - WT_RET(__wt_block_read_off(session, block, buf, logid, offset, size, checksum)); + WT_RET(__wt_block_read_off(session, block, buf, objectid, offset, size, checksum)); /* Optionally discard blocks from the system's buffer cache. */ WT_RET(__wt_block_discard(session, block, (size_t)size)); @@ -113,7 +113,7 @@ __wt_bm_read( * Dump a block into the log in 1KB chunks. */ static int -__wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t logid, wt_off_t offset, +__wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t objectid, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_ATTRIBUTE((cold)) { WT_DECL_ITEM(tmp); @@ -122,7 +122,7 @@ __wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t logid, wt_ #define WT_CORRUPT_FMT "{%" PRIu32 ": %" PRIuMAX ", %" PRIu32 ", %#" PRIx32 "}" if (buf->size == 0) { - __wt_errx(session, WT_CORRUPT_FMT ": empty buffer, no dump available", logid, + __wt_errx(session, WT_CORRUPT_FMT ": empty buffer, no dump available", objectid, (uintmax_t)offset, size, checksum); return (0); } @@ -134,7 +134,7 @@ __wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t logid, wt_ WT_ERR(__wt_buf_catfmt(session, tmp, "%02x ", ((uint8_t *)buf->data)[i])); if (++i == buf->size || i % 1024 == 0) { __wt_errx(session, - WT_CORRUPT_FMT ": (chunk %" WT_SIZET_FMT " of %" WT_SIZET_FMT "): %.*s", logid, + WT_CORRUPT_FMT ": (chunk %" WT_SIZET_FMT " of %" WT_SIZET_FMT "): %.*s", objectid, (uintmax_t)offset, size, checksum, ++chunk, nchunks, (int)tmp->size, (char *)tmp->data); if (i == buf->size) @@ -158,15 +158,15 @@ __wt_bm_corrupt(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_DECL_ITEM(tmp); WT_DECL_RET; wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; /* Read the block. */ WT_RET(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_bm_read(bm, session, tmp, addr, addr_size)); /* Crack the cookie, dump the block. */ - WT_ERR(__wt_block_buffer_to_addr(bm->block, addr, &logid, &offset, &size, &checksum)); - WT_ERR(__wt_bm_corrupt_dump(session, tmp, logid, offset, size, checksum)); + WT_ERR(__wt_block_buffer_to_addr(bm->block, addr, &objectid, &offset, &size, &checksum)); + WT_ERR(__wt_bm_corrupt_dump(session, tmp, objectid, offset, size, checksum)); err: __wt_scr_free(session, &tmp); @@ -211,39 +211,40 @@ err: * Get a block file handle. */ int -__wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, WT_FH **fhp) +__wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, WT_FH **fhp) { WT_DECL_ITEM(tmp); WT_DECL_RET; const char *filename; - if (!block->log_structured || logid == block->logid) { + if (!block->has_objects || objectid == block->objectid) { *fhp = block->fh; return (0); } /* TODO: tiered: fh readlock; we may want a reference count on each file handle given out. */ - if (logid * sizeof(WT_FILE_HANDLE *) < block->lfh_alloc && (*fhp = block->lfh[logid]) != NULL) + if (objectid * sizeof(WT_FILE_HANDLE *) < block->ofh_alloc && + (*fhp = block->ofh[objectid]) != NULL) return (0); /* TODO: tiered: fh writelock */ /* Ensure the array goes far enough. */ - WT_RET(__wt_realloc_def(session, &block->lfh_alloc, logid + 1, &block->lfh)); - if (logid >= block->max_logid) - block->max_logid = logid + 1; - if ((*fhp = block->lfh[logid]) != NULL) + WT_RET(__wt_realloc_def(session, &block->ofh_alloc, objectid + 1, &block->ofh)); + if (objectid >= block->max_objectid) + block->max_objectid = objectid + 1; + if ((*fhp = block->ofh[objectid]) != NULL) return (0); WT_RET(__wt_scr_alloc(session, 0, &tmp)); - if (logid == 0) + if (objectid == 0) filename = block->name; else { - WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, logid)); + WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, objectid)); filename = tmp->data; } WT_ERR(__wt_open(session, filename, WT_FS_OPEN_FILE_TYPE_DATA, - WT_FS_OPEN_READONLY | block->file_flags, &block->lfh[logid])); - *fhp = block->lfh[logid]; + WT_FS_OPEN_READONLY | block->file_flags, &block->ofh[objectid])); + *fhp = block->ofh[objectid]; WT_ASSERT(session, *fhp != NULL); err: @@ -256,7 +257,7 @@ err: * Read an addr/size pair referenced block into a buffer. */ int -__wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t logid, +__wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t objectid, wt_off_t offset, uint32_t size, uint32_t checksum) { WT_BLOCK_HEADER *blk, swap; @@ -293,7 +294,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uin block->name, size, block->allocsize); WT_RET(__wt_buf_init(session, buf, bufsize)); - WT_RET(__wt_block_fh(session, block, logid, &fh)); + WT_RET(__wt_block_fh(session, block, objectid, &fh)); WT_RET(__wt_read(session, fh, offset, size, buf->mem)); buf->size = size; @@ -327,7 +328,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uin block->name, size, (uintmax_t)offset, swap.checksum, checksum); if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) - WT_IGNORE_RET(__wt_bm_corrupt_dump(session, buf, logid, offset, size, checksum)); + WT_IGNORE_RET(__wt_bm_corrupt_dump(session, buf, objectid, offset, size, checksum)); /* Panic if a checksum fails during an ordinary read. */ F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); diff --git a/src/third_party/wiredtiger/src/block/block_slvg.c b/src/third_party/wiredtiger/src/block/block_slvg.c index 7e09d8ab147..280d520d1dd 100644 --- a/src/third_party/wiredtiger/src/block/block_slvg.c +++ b/src/third_party/wiredtiger/src/block/block_slvg.c @@ -104,13 +104,13 @@ __wt_block_salvage_next( WT_DECL_RET; WT_FH *fh; wt_off_t max, offset; - uint32_t allocsize, checksum, logid, size; + uint32_t allocsize, checksum, objectid, size; uint8_t *endp; *eofp = 0; - /* FIXME: salvage across all chunks in a log-structured tree. */ - logid = 0; + /* TODO: tiered: salvage across all objects in a tiered tree. */ + objectid = 0; fh = block->fh; allocsize = block->allocsize; @@ -140,19 +140,19 @@ __wt_block_salvage_next( * otherwise, move past it. */ if (!__wt_block_offset_invalid(block, offset, size) && - __wt_block_read_off(session, block, tmp, logid, offset, size, checksum) == 0) + __wt_block_read_off(session, block, tmp, objectid, offset, size, checksum) == 0) break; /* Free the allocation-size block. */ __wt_verbose(session, WT_VERB_SALVAGE, "skipping %" PRIu32 "B at file offset %" PRIuMAX, allocsize, (uintmax_t)offset); - WT_ERR(__wt_block_off_free(session, block, logid, offset, (wt_off_t)allocsize)); + WT_ERR(__wt_block_off_free(session, block, objectid, offset, (wt_off_t)allocsize)); block->slvg_off += allocsize; } /* Re-create the address cookie that should reference this block. */ endp = addr; - WT_ERR(__wt_block_addr_to_buffer(block, &endp, logid, offset, size, checksum)); + WT_ERR(__wt_block_addr_to_buffer(block, &endp, objectid, offset, size, checksum)); *addr_sizep = WT_PTRDIFF(endp, addr); done: @@ -170,7 +170,7 @@ __wt_block_salvage_valid( WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, size_t addr_size, bool valid) { wt_off_t offset; - uint32_t size, logid, checksum; + uint32_t size, objectid, checksum; WT_UNUSED(addr_size); @@ -178,11 +178,11 @@ __wt_block_salvage_valid( * Crack the cookie. If the upper layer took the block, move past it; if the upper layer * rejected the block, move past an allocation size chunk and free it. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); if (valid) block->slvg_off = offset + size; else { - WT_RET(__wt_block_off_free(session, block, logid, offset, (wt_off_t)block->allocsize)); + WT_RET(__wt_block_off_free(session, block, objectid, offset, (wt_off_t)block->allocsize)); block->slvg_off = offset + block->allocsize; } diff --git a/src/third_party/wiredtiger/src/block/block_tiered.c b/src/third_party/wiredtiger/src/block/block_tiered.c index d922a663e03..b275ccd95a7 100644 --- a/src/third_party/wiredtiger/src/block/block_tiered.c +++ b/src/third_party/wiredtiger/src/block/block_tiered.c @@ -34,8 +34,8 @@ __wt_block_tiered_load(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_BLOCK_CKPT * TODO: tiered: this call currently advances the object id, that's probably not appropriate for * readonly opens. Perhaps it's also not appropriate for opening at an older checkpoint? */ - if (block->log_structured) { - block->logid = ci->root_logid; + if (block->has_objects) { + block->objectid = ci->root_objectid; /* Advance to the next file for future changes. */ WT_RET(__wt_block_tiered_newfile(session, block)); @@ -64,8 +64,8 @@ __wt_block_tiered_newfile(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_ERR(__wt_close(session, &block->fh)); /* Bump to a new file ID. */ - ++block->logid; - WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, block->logid)); + ++block->objectid; + WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, block->objectid)); filename = tmp->data; WT_WITH_BUCKET_STORAGE(session->bucket_storage, session, { diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index 9785be9d12d..28c95415254 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -316,12 +316,12 @@ __wt_block_verify_addr( WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) { wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(addr_size); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); /* Add to the per-file list. */ WT_RET(__verify_filefrag_add(session, block, NULL, offset, size, false)); diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index 283b2f2a12d..ecbc3d02553 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -189,14 +189,14 @@ __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_ size_t *addr_sizep, bool data_checksum, bool checkpoint_io) { wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; uint8_t *endp; - WT_RET(__wt_block_write_off( - session, block, buf, &logid, &offset, &size, &checksum, data_checksum, checkpoint_io, false)); + WT_RET(__wt_block_write_off(session, block, buf, &objectid, &offset, &size, &checksum, + data_checksum, checkpoint_io, false)); endp = addr; - WT_RET(__wt_block_addr_to_buffer(block, &endp, logid, offset, size, checksum)); + WT_RET(__wt_block_addr_to_buffer(block, &endp, objectid, offset, size, checksum)); *addr_sizep = WT_PTRDIFF(endp, addr); return (0); @@ -207,7 +207,7 @@ __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_ * Write a buffer into a block, returning the block's offset, size and checksum. */ static int -__block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *logidp, +__block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) { @@ -216,7 +216,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 WT_FH *fh; wt_off_t offset; size_t align_size; - uint32_t checksum, logid; + uint32_t checksum, objectid; uint8_t *file_sizep; bool local_locked; @@ -225,7 +225,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 *checksump = 0; /* -Werror=maybe-uninitialized */ fh = block->fh; - logid = block->logid; + objectid = block->objectid; /* Buffers should be aligned for writing. */ if (!F_ISSET(buf, WT_ITEM_ALIGNED)) { @@ -327,7 +327,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 if ((ret = __wt_write(session, fh, offset, align_size, buf->mem)) != 0) { if (!caller_locked) __wt_spin_lock(session, &block->live_lock); - WT_TRET(__wt_block_off_free(session, block, logid, offset, (wt_off_t)align_size)); + WT_TRET(__wt_block_off_free(session, block, objectid, offset, (wt_off_t)align_size)); if (!caller_locked) __wt_spin_unlock(session, &block->live_lock); WT_RET(ret); @@ -361,7 +361,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 __wt_verbose(session, WT_VERB_WRITE, "off %" PRIuMAX ", size %" PRIuMAX ", checksum %#" PRIx32, (uintmax_t)offset, (uintmax_t)align_size, checksum); - *logidp = logid; + *objectidp = objectid; *offsetp = offset; *sizep = WT_STORE_SIZE(align_size); *checksump = checksum; @@ -374,7 +374,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 * Write a buffer into a block, returning the block's offset, size and checksum. */ int -__wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *logidp, +__wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) { @@ -386,8 +386,8 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, ui * never see anything other than their original content. */ __wt_page_header_byteswap(buf->mem); - ret = __block_write_off(session, block, buf, logidp, offsetp, sizep, checksump, data_checksum, - checkpoint_io, caller_locked); + ret = __block_write_off(session, block, buf, objectidp, offsetp, sizep, checksump, + data_checksum, checkpoint_io, caller_locked); __wt_page_header_byteswap(buf->mem); return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 5f3e03bde3e..b4ce5d86b4c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -660,7 +660,7 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - WT_ERR(__cursor_func_init(cbt, false)); + WT_ERR(__wt_cursor_func_init(cbt, false)); /* * If we aren't already iterating in the right direction, there's some setup to do. diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index abf31424525..b1bdfebffe1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -607,7 +607,7 @@ __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - WT_ERR(__cursor_func_init(cbt, false)); + WT_ERR(__wt_cursor_func_init(cbt, false)); /* * If we aren't already iterating in the right direction, there's some setup to do. diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 9449be9603a..3b489da2b89 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -532,7 +532,7 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) * (unlike other cursor functions), because we don't anticipate applications searching for a key * they currently have pinned.) */ - WT_ERR(__cursor_localkey(cursor)); + WT_ERR(__wt_cursor_localkey(cursor)); __cursor_novalue(cursor); __cursor_state_save(cursor, &state); @@ -555,7 +555,7 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) } } if (!valid) { - WT_ERR(__cursor_func_init(cbt, true)); + WT_ERR(__wt_cursor_func_init(cbt, true)); if (btree->type == BTREE_ROW) { WT_ERR(__cursor_row_search(cbt, false, NULL, NULL)); @@ -628,7 +628,7 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) * (unlike other cursor functions), because we don't anticipate applications searching for a key * they currently have pinned.) */ - WT_ERR(__cursor_localkey(cursor)); + WT_ERR(__wt_cursor_localkey(cursor)); __cursor_novalue(cursor); __cursor_state_save(cursor, &state); @@ -661,7 +661,7 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_ERR(__wt_cursor_valid(cbt, cbt->tmp, WT_RECNO_OOB, &valid)); } if (!valid) { - WT_ERR(__cursor_func_init(cbt, true)); + WT_ERR(__wt_cursor_func_init(cbt, true)); /* * Set the "insert" flag for row-store search; we may intend to position the cursor at the @@ -823,7 +823,7 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) * or value. (Restart could still use the pinned page, but that's an unlikely path.) Re-save * the cursor state: we may retry but eventually fail. */ - WT_TRET(__cursor_localkey(cursor)); + WT_TRET(__wt_cursor_localkey(cursor)); WT_TRET(__cursor_localvalue(cursor)); __cursor_state_save(cursor, &state); goto err; @@ -833,12 +833,12 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) * The pinned page goes away if we do a search, get a local copy of any pinned key or value. * Re-save the cursor state: we may retry but eventually fail. */ - WT_ERR(__cursor_localkey(cursor)); + WT_ERR(__wt_cursor_localkey(cursor)); WT_ERR(__cursor_localvalue(cursor)); __cursor_state_save(cursor, &state); retry: - WT_ERR(__cursor_func_init(cbt, true)); + WT_ERR(__wt_cursor_func_init(cbt, true)); if (btree->type == BTREE_ROW) { WT_ERR(__cursor_row_search(cbt, true, NULL, NULL)); @@ -965,11 +965,11 @@ __wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) * any pinned value. Unlike most of the btree cursor routines, we don't have to save/restore the * cursor key state, none of the work done here changes the cursor state. */ - WT_ERR(__cursor_localkey(cursor)); + WT_ERR(__wt_cursor_localkey(cursor)); __cursor_novalue(cursor); retry: - WT_ERR(__cursor_func_init(cbt, true)); + WT_ERR(__wt_cursor_func_init(cbt, true)); WT_ERR(__cursor_row_search(cbt, true, NULL, NULL)); /* Just check for conflicts. */ @@ -1058,11 +1058,11 @@ retry: * Any pinned page goes away if we do a search, including as a result of a restart. Get a local * copy of any pinned key and re-save the cursor state: we may retry but eventually fail. */ - WT_ERR(__cursor_localkey(cursor)); + WT_ERR(__wt_cursor_localkey(cursor)); __cursor_state_save(cursor, &state); searched = true; - WT_ERR(__cursor_func_init(cbt, true)); + WT_ERR(__wt_cursor_func_init(cbt, true)); if (btree->type == BTREE_ROW) { WT_ERR_NOTFOUND_OK(__cursor_row_search(cbt, false, NULL, NULL), true); @@ -1230,7 +1230,7 @@ __btcur_update(WT_CURSOR_BTREE *cbt, WT_ITEM *value, u_int modify_type) * or value. (Restart could still use the pinned page, but that's an unlikely path.) Re-save * the cursor state: we may retry but eventually fail. */ - WT_TRET(__cursor_localkey(cursor)); + WT_TRET(__wt_cursor_localkey(cursor)); WT_TRET(__cursor_localvalue(cursor)); __cursor_state_save(cursor, &state); goto err; @@ -1240,12 +1240,12 @@ __btcur_update(WT_CURSOR_BTREE *cbt, WT_ITEM *value, u_int modify_type) * The pinned page goes away if we do a search, get a local copy of any pinned key or value. * Re-save the cursor state: we may retry but eventually fail. */ - WT_ERR(__cursor_localkey(cursor)); + WT_ERR(__wt_cursor_localkey(cursor)); WT_ERR(__cursor_localvalue(cursor)); __cursor_state_save(cursor, &state); retry: - WT_ERR(__cursor_func_init(cbt, true)); + WT_ERR(__wt_cursor_func_init(cbt, true)); WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(cbt, true, NULL, NULL) : __cursor_col_search(cbt, NULL, NULL)); diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index c68855988a2..1093ed26b57 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -390,7 +390,7 @@ __wt_debug_offset( */ block = S2BT(session)->bm->block; endp = addr; - WT_RET(__wt_block_addr_to_buffer(block, &endp, block->logid, offset, size, checksum)); + WT_RET(__wt_block_addr_to_buffer(block, &endp, block->objectid, offset, size, checksum)); /* * Read the address through the btree I/O functions (so the block is decompressed as necessary). diff --git a/src/third_party/wiredtiger/src/btree/bt_random.c b/src/third_party/wiredtiger/src/btree/bt_random.c index 6a2af385822..25cf73fc0b9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_random.c +++ b/src/third_party/wiredtiger/src/btree/bt_random.c @@ -506,7 +506,7 @@ __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) * sampling, pick a roughly random leaf page in the tree and return an entry from it. */ if (cbt->ref == NULL || cbt->next_random_sample_size == 0) { - WT_ERR(__cursor_func_init(cbt, true)); + WT_ERR(__wt_cursor_func_init(cbt, true)); WT_WITH_PAGE_INDEX(session, ret = __wt_random_descent(session, &cbt->ref, read_flags)); if (ret == 0) { WT_ERR(__random_leaf(cbt)); diff --git a/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c b/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c index 3c191235020..3ca1dfed4ba 100644 --- a/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c +++ b/src/third_party/wiredtiger/src/checksum/arm64/crc32-arm64.c @@ -40,10 +40,13 @@ #define CRC32CX(crc, value) \ __asm__("crc32cx %w[c], %w[c], %x[v]" : [ c ] "+r"(*&crc) : [ v ] "r"(+value)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-macros" #define CRC32CW(crc, value) \ __asm__("crc32cw %w[c], %w[c], %w[v]" : [ c ] "+r"(*&crc) : [ v ] "r"(+value)) #define CRC32CH(crc, value) \ __asm__("crc32ch %w[c], %w[c], %w[v]" : [ c ] "+r"(*&crc) : [ v ] "r"(+value)) +#pragma GCC diagnostic pop #define CRC32CB(crc, value) \ __asm__("crc32cb %w[c], %w[c], %w[v]" : [ c ] "+r"(*&crc) : [ v ] "r"(+value)) diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 654abaf40d5..908e02e8688 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -48,7 +48,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = { {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, {"rollback_error", "int", NULL, "min=0,max=10M", NULL, 0}, {"slow_checkpoint", "boolean", NULL, NULL, NULL, 0}, - {"table_logging", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"table_logging", "boolean", NULL, NULL, NULL, 0}, + {"update_restore_evict", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_eviction_subconfigs[] = { {"threads_max", "int", NULL, "min=1,max=20", NULL, 0}, @@ -107,7 +108,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"compatibility", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"error_prefix", "string", NULL, NULL, NULL, 0}, {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2}, {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0}, @@ -270,7 +271,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -424,7 +425,7 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -469,7 +470,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_backup_info", "string", NULL, NULL, NULL, 0}, @@ -534,7 +535,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -581,7 +582,7 @@ static const WT_CONFIG_CHECK confchk_object_meta[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_backup_info", "string", NULL, NULL, NULL, 0}, @@ -645,7 +646,7 @@ static const WT_CONFIG_CHECK confchk_tier_meta[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"bucket", "string", NULL, NULL, NULL, 0}, {"bucket_prefix", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, @@ -756,7 +757,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -834,7 +835,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -911,7 +912,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -984,7 +985,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 9}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -1069,11 +1070,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),error_prefix=,eviction=(threads_max=8," - "threads_min=1),eviction_checkpoint_target=1," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,eviction_updates_target=0" - ",eviction_updates_trigger=0," + "table_logging=false,update_restore_evict=false),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",eviction_updates_target=0,eviction_updates_trigger=0," "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),history_store=(file_max=0)," "io_capacity=(total=0),log=(archive=true,os_cache_dirty_pct=0," @@ -1319,10 +1320,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "config_base=true,create=false,debug_mode=(checkpoint_retention=0" ",corruption_abort=true,cursor_copy=false,eviction=false," "log_retention=0,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "slow_checkpoint=false,table_logging=false," + "update_restore_evict=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," @@ -1354,10 +1355,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "config_base=true,create=false,debug_mode=(checkpoint_retention=0" ",corruption_abort=true,cursor_copy=false,eviction=false," "log_retention=0,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "slow_checkpoint=false,table_logging=false," + "update_restore_evict=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "exclusive=false,extensions=,file_close_sync=true,file_extend=," @@ -1390,9 +1391,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "table_logging=false,update_restore_evict=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," @@ -1424,9 +1426,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "debug_mode=(checkpoint_retention=0,corruption_abort=true," "cursor_copy=false,eviction=false,log_retention=0," "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "table_logging=false,update_restore_evict=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",eviction_updates_target=0,eviction_updates_trigger=0," "extensions=,file_close_sync=true,file_extend=," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 3d7b5fed416..b35d198c3aa 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1150,6 +1150,7 @@ err: * the sweep server. */ WT_TRET(__wt_sweep_destroy(session)); + WT_TRET(__wt_tiered_storage_destroy(session)); /* * Shut down the checkpoint and capacity server threads: we don't want to throttle writes and @@ -1961,6 +1962,11 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) else FLD_CLR(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.update_restore_evict", &cval)); + if (cval.val) + FLD_SET(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT); + else + FLD_CLR(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT); return (0); } diff --git a/src/third_party/wiredtiger/src/conn/conn_cache.c b/src/third_party/wiredtiger/src/conn/conn_cache.c index 53c4d4b1cc1..81fca88fe09 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache.c @@ -163,7 +163,7 @@ __cache_config_local(WT_SESSION_IMPL *session, bool shared, const char *cfg[]) * Configure or reconfigure the current cache and shared cache. */ int -__wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) +__wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig) { WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; @@ -178,10 +178,10 @@ __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) was_shared = F_ISSET(conn, WT_CONN_CACHE_POOL); /* Cleanup if reconfiguring */ - if (reconfigure && was_shared && !now_shared) + if (reconfig && was_shared && !now_shared) /* Remove ourselves from the pool if necessary */ WT_RET(__wt_conn_cache_pool_destroy(session)); - else if (reconfigure && !was_shared && now_shared) + else if (reconfig && !was_shared && now_shared) /* * Cache size will now be managed by the cache pool - the start size always needs to be zero * to allow the pool to manage how much memory is in-use. @@ -203,7 +203,7 @@ __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) * Resize the thread group if reconfiguring, otherwise the thread group will be initialized as * part of creating the cache. */ - if (reconfigure) + if (reconfig) WT_RET(__wt_thread_group_resize(session, &conn->evict_threads, conn->evict_threads_min, conn->evict_threads_max, WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL)); @@ -231,7 +231,7 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) cache = conn->cache; /* Use a common routine for run-time configuration options. */ - WT_RET(__wt_cache_config(session, false, cfg)); + WT_RET(__wt_cache_config(session, cfg, false)); /* * The lowest possible page read-generation has a special meaning, it marks a page for forcible diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 0c39475b207..5e56a841de4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -26,6 +26,9 @@ __conn_dhandle_config_clear(WT_SESSION_IMPL *session) __wt_free(session, *a); __wt_free(session, dhandle->cfg); __wt_free(session, dhandle->meta_base); +#ifdef HAVE_DIAGNOSTIC + __wt_free(session, dhandle->orig_meta_base); +#endif } /* @@ -85,6 +88,9 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) cfg[3] = NULL; WT_ERR(__wt_strdup(session, WT_CONFIG_BASE(session, file_meta), &dhandle->cfg[0])); WT_ASSERT(session, dhandle->meta_base == NULL); +#ifdef HAVE_DIAGNOSTIC + WT_ASSERT(session, dhandle->orig_meta_base == NULL); +#endif WT_ERR(__wt_config_collapse(session, cfg, &tmp)); /* * Now strip out the checkpoint related items from the configuration string and that is now @@ -108,6 +114,14 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session) } dhandle->cfg[1] = metaconf; dhandle->meta_base = base; + dhandle->meta_base_length = base == NULL ? 0 : strlen(base); +#ifdef HAVE_DIAGNOSTIC + /* Save the original metadata value for further check to avoid writing corrupted data. */ + if (base == NULL) + dhandle->orig_meta_base = NULL; + else + WT_ERR(__wt_strdup(session, base, &dhandle->orig_meta_base)); +#endif return (0); err: @@ -191,7 +205,7 @@ __wt_conn_dhandle_alloc(WT_SESSION_IMPL *session, const char *uri, const char *c WT_RET_PANIC(session, EINVAL, "illegal handle allocation URI %s", uri); /* Btree handles keep their data separate from the interface. */ - if (dhandle->type == WT_DHANDLE_TYPE_BTREE) { + if (WT_DHANDLE_BTREE(dhandle)) { WT_ERR(__wt_calloc_one(session, &btree)); dhandle->handle = btree; btree->dhandle = dhandle; @@ -295,7 +309,7 @@ __wt_conn_dhandle_close(WT_SESSION_IMPL *session, bool final, bool mark_dead) * The only data handle type that uses the "handle" field is btree. For other data handle types, * it should be NULL. */ - is_btree = dhandle->type == WT_DHANDLE_TYPE_BTREE; + is_btree = WT_DHANDLE_BTREE(dhandle); btree = is_btree ? dhandle->handle : NULL; if (is_btree) { @@ -507,7 +521,7 @@ __wt_conn_dhandle_open(WT_SESSION_IMPL *session, const char *cfg[], uint32_t fla WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_CLOSING_NO_MORE_OPENS)); /* Turn off eviction. */ - if (dhandle->type == WT_DHANDLE_TYPE_BTREE) + if (WT_DHANDLE_BTREE(dhandle)) WT_RET(__wt_evict_file_exclusive_on(session)); /* @@ -578,7 +592,7 @@ err: F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); } - if (dhandle->type == WT_DHANDLE_TYPE_BTREE) + if (WT_DHANDLE_BTREE(dhandle)) __wt_evict_file_exclusive_off(session); if (ret == ENOENT && F_ISSET(dhandle, WT_DHANDLE_IS_METADATA)) { @@ -690,8 +704,7 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, goto done; if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || F_ISSET(dhandle, WT_DHANDLE_DEAD) || - dhandle->type != WT_DHANDLE_TYPE_BTREE || dhandle->checkpoint != NULL || - WT_IS_METADATA(dhandle)) + !WT_DHANDLE_BTREE(dhandle) || dhandle->checkpoint != NULL || WT_IS_METADATA(dhandle)) continue; WT_ERR(__conn_btree_apply_internal(session, dhandle, file_func, name_func, cfg)); } diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index fb670dae9df..1e64aa61846 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -28,6 +28,7 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) TAILQ_INIT(&conn->encryptqh); /* Encryptor list */ TAILQ_INIT(&conn->extractorqh); /* Extractor list */ TAILQ_INIT(&conn->storagesrcqh); /* Storage source list */ + TAILQ_INIT(&conn->tieredqh); /* Tiered work unit list */ TAILQ_INIT(&conn->lsmqh); /* WT_LSM_TREE list */ @@ -54,6 +55,7 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); WT_SPIN_INIT_SESSION_TRACKED(session, &conn->schema_lock, schema); WT_RET(__wt_spin_init(session, &conn->storage_lock, "tiered storage")); + WT_RET(__wt_spin_init(session, &conn->tiered_lock, "tiered work unit list")); WT_RET(__wt_spin_init(session, &conn->turtle_lock, "turtle file")); /* Read-write locks */ @@ -120,6 +122,7 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->schema_lock); __wt_spin_destroy(session, &conn->storage_lock); __wt_rwlock_destroy(session, &conn->table_lock); + __wt_spin_destroy(session, &conn->tiered_lock); __wt_spin_destroy(session, &conn->turtle_lock); /* Free LSM serialization resources. */ diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 532975fc571..8012c5d84fc 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -603,9 +603,8 @@ __log_file_server(void *arg) */ if (conn->hot_backup_start == 0 && conn->log_cursors == 0) { WT_WITH_HOTBACKUP_READ_LOCK(session, - WT_ERR_ERROR_OK( - __wt_ftruncate(session, close_fh, close_end_lsn.l.offset), ENOTSUP, false), - NULL); + ret = __wt_ftruncate(session, close_fh, close_end_lsn.l.offset), NULL); + WT_ERR_ERROR_OK(ret, ENOTSUP, false); } WT_SET_LSN(&close_end_lsn, close_end_lsn.l.file + 1, 0); __wt_spin_lock(session, &log->log_sync_lock); diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 20467b4228b..b86ca4eb616 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -206,7 +206,7 @@ __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) * can know if statistics are enabled or not. */ WT_RET(__wt_statlog_create(session, cfg)); - WT_RET(__wt_tiered_storage_create(session, cfg, false)); + WT_RET(__wt_tiered_storage_create(session, cfg)); WT_RET(__wt_logmgr_create(session)); /* diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c index dae64bee291..c8d1330d7ff 100644 --- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c +++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c @@ -418,7 +418,7 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_ERR(ret); WT_ERR(__wt_conn_optrack_setup(session, cfg, true)); WT_ERR(__wt_conn_statistics_config(session, cfg)); - WT_ERR(__wt_cache_config(session, true, cfg)); + WT_ERR(__wt_cache_config(session, cfg, true)); WT_ERR(__wt_capacity_server_create(session, cfg)); WT_ERR(__wt_checkpoint_server_create(session, cfg)); WT_ERR(__wt_debug_mode_config(session, cfg)); @@ -426,7 +426,7 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_ERR(__wt_logmgr_reconfig(session, cfg)); WT_ERR(__wt_lsm_manager_reconfig(session, cfg)); WT_ERR(__wt_statlog_create(session, cfg)); - WT_ERR(__wt_tiered_storage_create(session, cfg, true)); + WT_ERR(__wt_tiered_conn_config(session, cfg, true)); WT_ERR(__wt_sweep_config(session, cfg)); WT_ERR(__wt_timing_stress_config(session, cfg)); WT_ERR(__wt_verbose_config(session, cfg)); diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index 90f050e161d..d68d3641666 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -61,7 +61,7 @@ __sweep_expire_one(WT_SESSION_IMPL *session) WT_DECL_RET; dhandle = session->dhandle; - btree = dhandle->type == WT_DHANDLE_TYPE_BTREE ? dhandle->handle : NULL; + btree = WT_DHANDLE_BTREE(dhandle) ? dhandle->handle : NULL; /* * Acquire an exclusive lock on the handle and mark it dead. diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c index 4d8a2ab5958..24a4a5a16f7 100644 --- a/src/third_party/wiredtiger/src/conn/conn_tiered.c +++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c @@ -120,24 +120,151 @@ err: } /* - * __tier_storage_copy -- - * Perform one iteration of copying newly flushed objects to the shared storage. + * __tier_flush_meta -- + * Perform one iteration of altering the metadata after a flush. This is in its own function so + * that we can hold the schema lock while doing the metadata tracking. */ static int -__tier_storage_copy(WT_SESSION_IMPL *session) +__tier_flush_meta( + WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *local_uri, const char *obj_uri) { + WT_DATA_HANDLE *dhandle; + WT_DECL_ITEM(buf); + WT_DECL_RET; + uint64_t now; + char *newconfig, *obj_value; + const char *cfg[3] = {NULL, NULL, NULL}; + bool tracking; + + tracking = false; + WT_RET(__wt_scr_alloc(session, 512, &buf)); + dhandle = &tiered->iface; + + newconfig = NULL; + WT_ERR(__wt_meta_track_on(session)); + tracking = true; + + WT_ERR(__wt_session_get_dhandle(session, dhandle->name, NULL, NULL, WT_DHANDLE_EXCLUSIVE)); /* - * Walk the work queue and copy file:<name> to shared storage object:<name>. Walk a tiered - * table's tiers array and copy it to any tier that allows WT_TIERS_OP_FLUSH. + * Once the flush call succeeds we want to first remove the file: entry from the metadata and + * then update the object: metadata to indicate the flush is complete. */ - /* XXX: We don't want to call this here, it is just to quiet the compiler that this function - * can return NULL. So it is a placeholder until we have real content here. + WT_ERR(__wt_metadata_remove(session, local_uri)); + WT_ERR(__wt_metadata_search(session, obj_uri, &obj_value)); + __wt_seconds(session, &now); + WT_ERR(__wt_buf_fmt(session, buf, "flush=%" PRIu64, now)); + cfg[0] = obj_value; + cfg[1] = buf->mem; + WT_ERR(__wt_config_collapse(session, cfg, &newconfig)); + WT_ERR(__wt_metadata_update(session, obj_uri, newconfig)); + WT_ERR(__wt_meta_track_off(session, true, ret != 0)); + tracking = false; + +err: + __wt_free(session, newconfig); + WT_TRET(__wt_session_release_dhandle(session)); + __wt_scr_free(session, &buf); + if (tracking) + WT_TRET(__wt_meta_track_off(session, true, ret != 0)); + return (ret); +} + +/* + * __wt_tier_do_flush -- + * Perform one iteration of copying newly flushed objects to the shared storage. + */ +int +__wt_tier_do_flush( + WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *local_uri, const char *obj_uri) +{ + WT_DECL_RET; + WT_FILE_SYSTEM *bucket_fs; + WT_STORAGE_SOURCE *storage_source; + const char *local_name, *obj_name; + + storage_source = tiered->bstorage->storage_source; + bucket_fs = tiered->bstorage->file_system; + + local_name = local_uri; + WT_PREFIX_SKIP_REQUIRED(session, local_name, "file:"); + obj_name = obj_uri; + WT_PREFIX_SKIP_REQUIRED(session, obj_name, "object:"); + + /* This call make take a while, and may fail due to network timeout. */ + WT_RET(storage_source->ss_flush( + storage_source, &session->iface, bucket_fs, local_name, obj_name, NULL)); + + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __tier_flush_meta(session, tiered, local_uri, obj_uri))); + WT_RET(ret); + + /* + * We may need a way to cleanup flushes for those not completed (after a crash), or failed (due + * to previous network outage). */ - WT_RET(__tier_storage_remove_local(session, NULL, 0)); + WT_RET(storage_source->ss_flush_finish( + storage_source, &session->iface, bucket_fs, local_name, obj_name, NULL)); return (0); } /* + * __wt_tier_flush -- + * Given an ID generate the URI names and call the flush code. + */ +int +__wt_tier_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint64_t id) +{ + WT_DECL_RET; + const char *local_uri, *obj_uri; + + local_uri = obj_uri = NULL; + WT_ERR(__wt_tiered_name(session, &tiered->iface, id, WT_TIERED_NAME_LOCAL, &local_uri)); + WT_ERR(__wt_tiered_name(session, &tiered->iface, id, WT_TIERED_NAME_OBJECT, &obj_uri)); + WT_ERR(__wt_tier_do_flush(session, tiered, local_uri, obj_uri)); + +err: + __wt_free(session, local_uri); + __wt_free(session, obj_uri); + return (ret); +} + +/* + * __tier_storage_copy -- + * Perform one iteration of copying newly flushed objects to the shared storage. + */ +static int +__tier_storage_copy(WT_SESSION_IMPL *session) +{ + WT_DECL_RET; + WT_TIERED_WORK_UNIT *entry; + + entry = NULL; + for (;;) { + /* + * We probably need some kind of flush generation so that we don't process flush items for + * tables that are added during an in-progress flush_tier. This thread could run due to a + * condition timeout rather than a signal. Checking that generation number would be part of + * calling __wt_tiered_get_flush so that we don't pull it off the queue until we're sure we + * want to process it. + */ + __wt_tiered_get_flush(session, &entry); + if (entry == NULL) + break; + WT_ERR(__wt_tier_flush(session, entry->tiered, entry->id)); + /* + * We are responsible for freeing the work unit when we're done with it. + */ + __wt_free(session, entry); + entry = NULL; + } + +err: + if (entry != NULL) + __wt_free(session, entry); + return (ret); +} + +/* * __tier_storage_remove -- * Perform one iteration of tiered storage local tier removal. */ @@ -238,6 +365,8 @@ __tiered_server(void *arg) WT_DECL_RET; WT_ITEM path, tmp; WT_SESSION_IMPL *session; + uint64_t cond_time, time_start, time_stop, timediff; + bool signalled; session = arg; conn = S2C(session); @@ -245,21 +374,31 @@ __tiered_server(void *arg) WT_CLEAR(path); WT_CLEAR(tmp); + /* Condition timeout is in microseconds. */ + cond_time = WT_MINUTE * WT_MILLION; + time_start = __wt_clock(session); + signalled = false; for (;;) { /* Wait until the next event. */ - __wt_cond_wait(session, conn->tiered_cond, WT_MINUTE, __tiered_server_run_chk); + __wt_cond_wait_signal( + session, conn->tiered_cond, cond_time, __tiered_server_run_chk, &signalled); /* Check if we're quitting or being reconfigured. */ if (!__tiered_server_run_chk(session)) break; + time_stop = __wt_clock(session); + timediff = WT_CLOCKDIFF_SEC(time_stop, time_start); /* * Here is where we do work. Work we expect to do: * - Copy any files that need moving from a flush tier call. * - Remove any cached objects that are aged out. */ - WT_ERR(__tier_storage_copy(session)); - WT_ERR(__tier_storage_remove(session, false)); + if (timediff >= WT_MINUTE || signalled) { + WT_ERR(__tier_storage_copy(session)); + WT_ERR(__tier_storage_remove(session, false)); + } + time_start = time_stop; } if (0) { @@ -316,7 +455,8 @@ __tiered_mgr_server(void *arg) /* * Here is where we do work. Work we expect to do: */ - __flush_tier_once(session, false); + WT_WITH_SCHEMA_LOCK(session, ret = __flush_tier_once(session, false)); + WT_ERR(ret); WT_ERR(__tier_storage_remove(session, false)); } @@ -339,7 +479,7 @@ __tiered_mgr_start(WT_CONNECTION_IMPL *conn) FLD_SET(conn->server_flags, WT_CONN_SERVER_TIERED_MGR); WT_RET(__wt_open_internal_session( - conn, "storage-mgr-server", true, 0, 0, &conn->tiered_mgr_session)); + conn, "storage-mgr-server", false, 0, 0, &conn->tiered_mgr_session)); session = conn->tiered_mgr_session; WT_RET(__wt_cond_alloc(session, "storage server", &conn->tiered_mgr_cond)); @@ -355,7 +495,7 @@ __tiered_mgr_start(WT_CONNECTION_IMPL *conn) * Start the tiered storage subsystem. */ int -__wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig) +__wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -364,20 +504,15 @@ __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[], bool rec conn = S2C(session); start = false; - /* Destroy any existing thread since we could be a reconfigure. */ - WT_RET(__wt_tiered_storage_destroy(session)); - if (reconfig) - WT_RET(__wt_tiered_conn_config(session, cfg, reconfig)); WT_RET(__tiered_manager_config(session, cfg, &start)); /* Start the internal thread. */ + WT_ERR(__wt_cond_alloc(session, "storage server", &conn->tiered_cond)); FLD_SET(conn->server_flags, WT_CONN_SERVER_TIERED); WT_ERR(__wt_open_internal_session(conn, "storage-server", true, 0, 0, &conn->tiered_session)); session = conn->tiered_session; - WT_ERR(__wt_cond_alloc(session, "storage server", &conn->tiered_cond)); - /* Start the thread. */ WT_ERR(__wt_thread_create(session, &conn->tiered_tid, __tiered_server, session)); conn->tiered_tid_set = true; @@ -388,6 +523,7 @@ __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[], bool rec if (0) { err: + FLD_CLR(conn->server_flags, WT_CONN_SERVER_TIERED); WT_TRET(__wt_tiered_storage_destroy(session)); } return (ret); @@ -402,6 +538,7 @@ __wt_tiered_storage_destroy(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_TIERED_WORK_UNIT *entry; conn = S2C(session); @@ -411,6 +548,10 @@ __wt_tiered_storage_destroy(WT_SESSION_IMPL *session) __wt_cond_signal(session, conn->tiered_cond); WT_TRET(__wt_thread_join(session, &conn->tiered_tid)); conn->tiered_tid_set = false; + while ((entry = TAILQ_FIRST(&conn->tieredqh)) != NULL) { + TAILQ_REMOVE(&conn->tieredqh, entry, q); + __wt_free(session, entry); + } } __wt_cond_destroy(session, &conn->tiered_cond); if (conn->tiered_session != NULL) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 046a7ac52db..04e53e5b4f2 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -591,7 +591,7 @@ __curfile_reopen_int(WT_CURSOR *cursor) */ if (ret == 0) { /* Assert a valid tree (we didn't race with eviction). */ - WT_ASSERT(session, dhandle->type == WT_DHANDLE_TYPE_BTREE); + WT_ASSERT(session, WT_DHANDLE_BTREE(dhandle)); WT_ASSERT(session, ((WT_BTREE *)dhandle->handle)->root.page != NULL); btree = CUR2BT(cursor); @@ -821,7 +821,7 @@ __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, c if (bulk) LF_SET(WT_BTREE_BULK | WT_DHANDLE_EXCLUSIVE); - WT_ASSERT(session, WT_PREFIX_MATCH(uri, "file:")); + WT_ASSERT(session, WT_PREFIX_MATCH(uri, "file:") || WT_PREFIX_MATCH(uri, "tiered:")); /* Get the handle and lock it while the cursor is using it. */ /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_hs.c b/src/third_party/wiredtiger/src/cursor/cur_hs.c index 74dd2899fe0..b494090af14 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_hs.c +++ b/src/third_party/wiredtiger/src/cursor/cur_hs.c @@ -147,6 +147,46 @@ __curhs_set_value_ptr(WT_CURSOR *hs_cursor, WT_CURSOR *file_cursor) } /* + * __curhs_search -- + * Search the history store for a given key and position the cursor on it. + */ +static int +__curhs_search(WT_CURSOR_BTREE *hs_cbt, bool insert) +{ + WT_BTREE *hs_btree; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + hs_btree = CUR2BT(hs_cbt); + session = CUR2S(hs_cbt); + +#ifdef HAVE_DIAGNOSTIC + /* + * Turn off cursor-order checks in all cases on search. The search/search-near functions turn + * them back on after a successful search. + */ + __wt_cursor_key_order_reset(hs_cbt); +#endif + + WT_ERR(__wt_cursor_localkey(&hs_cbt->iface)); + + WT_ERR(__wt_cursor_func_init(hs_cbt, true)); + + WT_WITH_BTREE(session, hs_btree, + ret = __wt_row_search(hs_cbt, &hs_cbt->iface.key, insert, NULL, false, NULL)); + +#ifdef HAVE_DIAGNOSTIC + WT_TRET(__wt_cursor_key_order_init(hs_cbt)); +#endif + +err: + if (ret != 0) + WT_TRET(__cursor_reset(hs_cbt)); + + return (ret); +} + +/* * __curhs_next -- * WT_CURSOR->next method for the history store cursor type. */ @@ -651,9 +691,17 @@ __curhs_search_near(WT_CURSOR *cursor, int *exactp) break; /* - * We are now smaller than the key range, which indicates nothing is visible to - * us in the specified key range. + * We're comparing the entire history store key (as opposed to just the data + * store component) because ordering can be different between the data store and + * history store due to packing. Since we know we're NOT in the specified key + * range due to the check above, checking whether we're before or after the full + * history store key that we're running a `search near` on will tell us whether + * we're before or after the specified key range. + * + * If we're before the specified key range, that means nothing is visible to us + * in the specified key range and we should terminate the search. */ + WT_ERR(__wt_compare(session, NULL, &file_cursor->key, srch_key, &cmp)); if (cmp < 0) { ret = WT_NOTFOUND; goto err; @@ -719,9 +767,17 @@ __curhs_search_near(WT_CURSOR *cursor, int *exactp) break; /* - * We are now larger than the key range, which indicates nothing is visible to - * us in the specified key range. + * We're comparing the entire history store key (as opposed to just the data + * store component) because ordering can be different between the data store and + * history store due to packing. Since we know we're NOT in the specified key + * range due to the check above, checking whether we're before or after the full + * history store key that we're running a `search near` on will tell us whether + * we're before or after the specified key range. + * + * If we're after the specified key range, that means nothing is visible to us + * in the specified key range and we should terminate the search. */ + WT_ERR(__wt_compare(session, NULL, &file_cursor->key, srch_key, &cmp)); if (cmp > 0) { ret = WT_NOTFOUND; goto err; @@ -860,16 +916,22 @@ __curhs_insert(WT_CURSOR *cursor) retry: /* Search the page and insert the updates. */ - WT_WITH_PAGE_INDEX(session, ret = __wt_hs_row_search(cbt, &file_cursor->key, true)); + WT_WITH_PAGE_INDEX(session, ret = __curhs_search(cbt, true)); WT_ERR(ret); ret = __wt_hs_modify(cbt, hs_upd); if (ret == WT_RESTART) goto retry; WT_ERR(ret); + /* + * Mark the insert as successful. Even if one of the calls below fails, some callers will still + * need to know whether the actual insert went through or not. + */ + hs_cursor->insert_success = true; + #ifdef HAVE_DIAGNOSTIC /* Do a search again and call next to check the key order. */ - WT_WITH_PAGE_INDEX(session, ret = __wt_hs_row_search(cbt, &file_cursor->key, true)); + WT_WITH_PAGE_INDEX(session, ret = __curhs_search(cbt, true)); WT_ASSERT(session, ret == 0); WT_ERR_NOTFOUND_OK(__curhs_file_cursor_next(session, file_cursor), false); #endif @@ -925,7 +987,7 @@ __curhs_remove(WT_CURSOR *cursor) hs_tombstone->txnid = WT_TXN_NONE; hs_tombstone->start_ts = hs_tombstone->durable_ts = WT_TS_NONE; while ((ret = __wt_hs_modify(cbt, hs_tombstone)) == WT_RESTART) { - WT_WITH_PAGE_INDEX(session, ret = __wt_hs_row_search(cbt, &file_cursor->key, false)); + WT_WITH_PAGE_INDEX(session, ret = __curhs_search(cbt, false)); WT_ERR(ret); } @@ -957,19 +1019,16 @@ __curhs_update(WT_CURSOR *cursor) WT_DECL_RET; WT_SESSION_IMPL *session; WT_UPDATE *hs_tombstone, *hs_upd; - bool retry; hs_cursor = (WT_CURSOR_HS *)cursor; file_cursor = hs_cursor->file_cursor; cbt = (WT_CURSOR_BTREE *)file_cursor; hs_tombstone = hs_upd = NULL; - retry = false; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, update, CUR2BT(file_cursor)); - /* We are assuming that the caller has already searched and found the key. */ - WT_ASSERT( - session, F_ISSET(file_cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET | WT_CURSTD_VALUE_INT)); + /* Update must be called with cursor positioned. */ + WT_ASSERT(session, F_ISSET(file_cursor, WT_CURSTD_KEY_INT)); WT_ASSERT(session, F_ISSET(hs_cursor, WT_HS_CUR_COUNTER_SET | WT_HS_CUR_TS_SET)); /* @@ -1000,15 +1059,8 @@ __curhs_update(WT_CURSOR *cursor) cbt->compare = 0; /* Make the updates and if we fail, search and try again. */ while ((ret = __wt_hs_modify(cbt, hs_tombstone)) == WT_RESTART) { - WT_WITH_PAGE_INDEX(session, ret = __wt_hs_row_search(cbt, &file_cursor->key, false)); + WT_WITH_PAGE_INDEX(session, ret = __curhs_search(cbt, false)); WT_ERR(ret); - retry = true; - } - - /* If we retry, search again to point to the updated value. */ - if (retry) { - WT_WITH_PAGE_INDEX(session, ret = __wt_hs_row_search(cbt, &file_cursor->key, false)); - WT_TRET(ret); } __curhs_set_key_ptr(cursor, file_cursor); @@ -1082,3 +1134,34 @@ err: } return (ret); } + +/* + * __wt_curhs_clear_insert_success -- + * Clear the insertion flag for the history store cursor. We should call this prior to using the + * WT_CURSOR->insert method. + */ +void +__wt_curhs_clear_insert_success(WT_CURSOR *cursor) +{ + WT_CURSOR_HS *hs_cursor; + + hs_cursor = (WT_CURSOR_HS *)cursor; + hs_cursor->insert_success = false; +} + +/* + * __wt_curhs_check_insert_success -- + * Check whether the insertion flag for the history store cursor is set or not. This signals + * whether or not the last WT_CURSOR->insert call successfully inserted the history store + * record. This is distinctly different from the return value of WT_CURSOR->insert since the + * return value could be non-zero due to cursor operations AFTER the actual history store + * insertion. + */ +bool +__wt_curhs_check_insert_success(WT_CURSOR *cursor) +{ + WT_CURSOR_HS *hs_cursor; + + hs_cursor = (WT_CURSOR_HS *)cursor; + return (hs_cursor->insert_success); +} diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 19a50939a7a..26d6915724a 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -1222,6 +1222,10 @@ __wt_cursor_init( cursor->modify == __wt_cursor_modify_value_format_notsup) cursor->modify = __cursor_modify; + /* Tiered cursors are not yet candidates for caching. */ + if (uri != NULL && WT_PREFIX_MATCH(uri, "tiered:")) + F_CLR(cursor, WT_CURSTD_CACHEABLE); + /* * Cursors that are internal to some other cursor (such as file cursors inside a table cursor) * should be closed after the containing cursor. Arrange for that to happen by putting internal diff --git a/src/third_party/wiredtiger/src/docs/build-autoconf.dox b/src/third_party/wiredtiger/src/docs/build-autoconf.dox new file mode 100644 index 00000000000..30310e2fcfe --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/build-autoconf.dox @@ -0,0 +1,205 @@ +/*! @page build-autoconf Building and installing WiredTiger with autoconf/libtool + +@section autoconf_github Building using Git and GitHub + +Skip this step if you are building from a WiredTiger release package, +and proceed with @ref autoconf_building. + +To build from the WiredTiger GitHub repository requires +<a href="http://git-scm.com/">git</a>, +<a href="http://www.gnu.org/software/autoconf/autoconf.html">autoconf</a>, +<a href="http://www.gnu.org/software/autoconf/automake.html">automake</a>, +<a href="http://www.gnu.org/software/libtool/libtool.html">libtool</a> and +related tools. The standard options for those tools can be specified +when configuring and building WiredTiger. + +First, clone the repository: + +@code +git clone git://github.com/wiredtiger/wiredtiger.git +@endcode + +Second, run \c autogen.sh to create the \c configure script: + +@code +cd wiredtiger +sh autogen.sh +@endcode + +Now proceed with @ref autoconf_building. + +@section autoconf_building Building WiredTiger + +To build the WiredTiger software on a POSIX system, change directory to +the top-level directory, then configure and build the software: + +@code +cd wiredtiger +./configure && make +@endcode + +To rebuild from scratch, discard any previous configuration by cleaning +out the build area: + +@code +make distclean +@endcode + +To see additional configuration options, run: + +@code +./configure --help +@endcode + +WiredTiger uses +<a href="http://www.gnu.org/software/autoconf/autoconf.html">autoconf</a> +<a href="http://www.gnu.org/software/autoconf/automake.html">automake</a>, +and <a href="http://www.gnu.org/software/libtool/libtool.html">libtool</a> +to create the configure script and Makefiles. The standard options for those +tools can be specified when configuring and building WiredTiger. + +@section autoconf_installing Installing WiredTiger + +The WiredTiger software consists of a library and a single standalone +utility. + +WiredTiger's distribution follows the GNU Coding Standards installation +guidelines, and by default WiredTiger builds and installs four versions +of the library in <code>/usr/local/lib</code>. For example: + +@code +file /usr/local/lib/libwiredtiger* +/usr/local/lib/libwiredtiger-1.0.0.so: ELF 64-bit LSB shared object, x86-64, version 1 (FreeBSD), dynamically linked, not stripped +/usr/local/lib/libwiredtiger.a: current ar archive +/usr/local/lib/libwiredtiger.la: libtool library file +/usr/local/lib/libwiredtiger.so: symbolic link to `libwiredtiger-1.0.0.so' +@endcode + +WiredTiger uses +<a href="http://www.gnu.org/software/libtool/libtool.html">libtool</a> to +build the libraries. By default, both shared and static libraries are built. +To build only static libraries, configure WiredTiger using the +\c --disable-shared argument. To build only shared libraries, configure using +WiredTiger using the \c --disable-static argument. + +In addition, WiredTiger installs a standalone utility program named +<code>wt</code>. By default, this utility is installed in +<code>/usr/local/bin/wt</code>. + +To install WiredTiger: + +@code +make install +@endcode + +To uninstall WiredTiger: + +@code +make uninstall +@endcode + +To install WiredTiger's libraries or binaries into alternate locations, +use the configuration or installation options described in the +<a href="http://www.gnu.org/prep/standards/">GNU coding standards</a> +documentation. For example, to install the libraries and binaries into +a different location: + +@code +./configure --prefix=/c/wiredtiger +@endcode + +@section autoconf_configure Configuring WiredTiger + +The WiredTiger software supports some additional configuration options: + +@par \c --enable-attach +Configure WiredTiger to sleep and wait for a debugger to attach on failure. +<b>DO NOT</b> configure this option in production environments. + +@par \c --enable-diagnostic +Configure WiredTiger to perform various run-time diagnostic tests. +<b>DO NOT</b> configure this option in production environments. + +@par \c --enable-lz4 +Configure WiredTiger for <a href="https://github.com/Cyan4973/lz4">LZ4</a> +compression; see @ref compression for more information. + +@par \c --enable-python +Build the WiredTiger <a href="http://www.python.org">Python</a> API; +requires <a href="http://swig.org">SWIG</a>. + +@par \c --enable-snappy +Configure WiredTiger for <a href="http://code.google.com/p/snappy/">snappy</a> +compression; see @ref compression for more information. + +@par \c --enable-zlib +Configure WiredTiger for <a href="http://www.zlib.net/">zlib</a> +compression; see @ref compression for more information. + +@par \c --enable-zstd +Configure WiredTiger for <a href="https://github.com/facebook/zstd">Zstd</a> +compression; see @ref compression for more information. + +@par \c --disable-standalone-build +Configure WiredTiger to disable standalone build. Standalone build is enabled +by default. + +@par <code>--with-builtins</code> +Configure WiredTiger to include support for extensions in the main library. +This avoids requiring additional libraries for supported extensions. Currently +supported options are \c lz4, \c snappy, \c zlib and \c zstd. + +@par <code>--with-python-prefix</code> +Configure WiredTiger to install Python libraries to a non-standard Python +install location. + +@par <code>--with-spinlock[=pthread, pthread_adaptive, gcc]</code> +Configure WiredTiger to use a specific mutex type for serialization; +options are \c pthread (the default, which configures WiredTiger to use +POSIX 1003.1c pthread mutexes), \c pthread_adaptive (which configures +WiredTiger to use POSIX 1003.1c pthread mutexes configured to be +adaptive (where that functionality is available), or \c gcc (which +configures WiredTiger to use gcc-based spinlocks). + +@section autoconf_compiler Changing compiler or loader options + +To change the compiler or loader behavior during the build, use the +<code>CC</code>, <code>CFLAGS</code>, <code>LDFLAGS</code>, or +<code>LIBS</code> environment variables: + +@par \c CC +The compiler. +@par \c CFLAGS +Compiler flags. +@par \c LDFLAGS +Loader flags. +@par \c LIBS +Additional libraries. + +For example, to specify a different compiler: + +@code +env CC=mygcc ./configure +@endcode + +By default, WiredTiger builds with the \c -O3 compiler optimization flag +unless the \c --enable-debug configuration option is specified, in which +case the \c -g compiler flag is used instead. For example, to specify +a different level of optimization: + +@code +env CFLAGS=-Os ./configure +@endcode + +To specify a different set of include files: + +@code +env CFLAGS=-I/usr/local/include ./configure +@endcode + +To specify an additional library: + +@code +env LIBS="-lrtf -lmin" LDFLAGS=-L/usr/local/lib ./configure +@endcode + */ diff --git a/src/third_party/wiredtiger/src/docs/build-posix.dox b/src/third_party/wiredtiger/src/docs/build-posix.dox index e0eb1577b29..7fd9cfbec14 100644 --- a/src/third_party/wiredtiger/src/docs/build-posix.dox +++ b/src/third_party/wiredtiger/src/docs/build-posix.dox @@ -1,5 +1,7 @@ /*! @page build-posix Building and installing WiredTiger on POSIX +To read instructions on using the legacy autoconf build system, see @subpage build-autoconf. + @section posix_github Building using Git and GitHub Skip this step if you are building from a WiredTiger release package, @@ -7,11 +9,10 @@ and proceed with @ref posix_building. To build from the WiredTiger GitHub repository requires <a href="http://git-scm.com/">git</a>, -<a href="http://www.gnu.org/software/autoconf/autoconf.html">autoconf</a>, -<a href="http://www.gnu.org/software/autoconf/automake.html">automake</a>, -<a href="http://www.gnu.org/software/libtool/libtool.html">libtool</a> and -related tools. The standard options for those tools can be specified -when configuring and building WiredTiger. +<a href="https://cmake.org/">CMake</a>, +<a href="https://ninja-build.org/">Ninja</a>. +We also suggest <a href="https://ccache.dev/">ccache</a> for improved build times +and <a href="http://www.swig.org/">SWIG</a> for building with Python support. First, clone the repository: @@ -19,68 +20,48 @@ First, clone the repository: git clone git://github.com/wiredtiger/wiredtiger.git @endcode -Second, run \c autogen.sh to create the \c configure script: - -@code -cd wiredtiger -sh autogen.sh -@endcode - Now proceed with @ref posix_building. @section posix_building Building WiredTiger To build the WiredTiger software on a POSIX system, change directory to -the top-level directory, then configure and build the software: +the top-level directory, then configure and create a new directory to run +your build from: @code cd wiredtiger -./configure && make +mkdir build +cd build @endcode -To rebuild from scratch, discard any previous configuration by cleaning -out the build area: - +Change into your newly created build directory and using CMake run the build configuration +step to generate your ninja build. @code -make distclean +cmake -G Ninja ../. @endcode -To see additional configuration options, run: - +Lastly, in the same directory you configured your build, run the ninja command to start the build: @code -./configure --help +ninja @endcode -WiredTiger uses -<a href="http://www.gnu.org/software/autoconf/autoconf.html">autoconf</a> -<a href="http://www.gnu.org/software/autoconf/automake.html">automake</a>, -and <a href="http://www.gnu.org/software/libtool/libtool.html">libtool</a> -to create the configure script and Makefiles. The standard options for those -tools can be specified when configuring and building WiredTiger. - @section posix_installing Installing WiredTiger -The WiredTiger software consists of a library and a single standalone -utility. +The WiredTiger software consists of a library and a single standalone utility. -WiredTiger's distribution follows the GNU Coding Standards installation -guidelines, and by default WiredTiger builds and installs four versions -of the library in <code>/usr/local/lib</code>. For example: +WiredTiger's distribution by default builds and installs the shared version library in <code>/usr/local/lib</code>. For example: @code file /usr/local/lib/libwiredtiger* -/usr/local/lib/libwiredtiger-1.0.0.so: ELF 64-bit LSB shared object, x86-64, version 1 (FreeBSD), dynamically linked, not stripped -/usr/local/lib/libwiredtiger.a: current ar archive -/usr/local/lib/libwiredtiger.la: libtool library file -/usr/local/lib/libwiredtiger.so: symbolic link to `libwiredtiger-1.0.0.so' +/usr/local/lib/libwiredtiger.so.10.0.1: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped +/usr/local/lib/libwiredtiger.so: symbolic link to `libwiredtiger.so.10.0.1' @endcode WiredTiger uses -<a href="http://www.gnu.org/software/libtool/libtool.html">libtool</a> to -build the libraries. By default, both shared and static libraries are built. -To build only static libraries, configure WiredTiger using the -\c --disable-shared argument. To build only shared libraries, configure using -WiredTiger using the \c --disable-static argument. +<a href="https://cmake.org/">CMake</a> to build and install the libraries. +By default, the shared libraries are built. To build only static +libraries, configure WiredTiger using the +\c -DENABLE_STATIC=1 argument. In addition, WiredTiger installs a standalone utility program named <code>wt</code>. By default, this utility is installed in @@ -89,71 +70,67 @@ In addition, WiredTiger installs a standalone utility program named To install WiredTiger: @code -make install +ninja install @endcode To uninstall WiredTiger: @code -make uninstall +xargs rm < install_manifest.txt @endcode +The \c install_manifest.txt file is created when you run <code>ninja install</code>. + To install WiredTiger's libraries or binaries into alternate locations, -use the configuration or installation options described in the -<a href="http://www.gnu.org/prep/standards/">GNU coding standards</a> -documentation. For example, to install the libraries and binaries into -a different location: +configure WiredTiger using the \c -DCMAKE_INSTALL_PREFIX=custom_path argument. +For example, to install the libraries and binaries into a different location: @code -./configure --prefix=/c/wiredtiger +cmake -DCMAKE_INSTALL_PREFIX=/c/wiredtiger -G Ninja ../. @endcode @section posix_configure Configuring WiredTiger The WiredTiger software supports some additional configuration options: -@par \c --enable-attach +@par \c -DHAVE_ATTACH=1 Configure WiredTiger to sleep and wait for a debugger to attach on failure. <b>DO NOT</b> configure this option in production environments. -@par \c --enable-diagnostic +@par \c -DHAVE_DIAGNOSTIC=1 Configure WiredTiger to perform various run-time diagnostic tests. <b>DO NOT</b> configure this option in production environments. -@par \c --enable-lz4 +@par \c -DENABLE_LZ4=1 Configure WiredTiger for <a href="https://github.com/Cyan4973/lz4">LZ4</a> compression; see @ref compression for more information. -@par \c --enable-python +@par \c -DENABLE_PYTHON=1 Build the WiredTiger <a href="http://www.python.org">Python</a> API; requires <a href="http://swig.org">SWIG</a>. -@par \c --enable-snappy +@par \c -DENABLE_SNAPPY=1 Configure WiredTiger for <a href="http://code.google.com/p/snappy/">snappy</a> compression; see @ref compression for more information. -@par \c --enable-zlib +@par \c -DENABLE_ZLIB=1 Configure WiredTiger for <a href="http://www.zlib.net/">zlib</a> compression; see @ref compression for more information. -@par \c --enable-zstd +@par \c -DENABLE_ZSTD=1 Configure WiredTiger for <a href="https://github.com/facebook/zstd">Zstd</a> compression; see @ref compression for more information. -@par \c --disable-standalone-build +@par \c -DWT_STANDALONE_BUILD=0 Configure WiredTiger to disable standalone build. Standalone build is enabled by default. -@par <code>--with-builtins</code> +@par \c -DHAVE_BUILTIN_EXTENSION_LZ4=1, \c -DHAVE_BUILTIN_EXTENSION_SNAPPY=1, \c -DHAVE_BUILTIN_EXTENSION_ZLIB=1, \c -DHAVE_BUILTIN_EXTENSION_ZSTD=1 Configure WiredTiger to include support for extensions in the main library. This avoids requiring additional libraries for supported extensions. Currently -supported options are \c lz4, \c snappy, \c zlib and \c zstd. +supported builtin options are for \c lz4, \c snappy, \c zlib and \c zstd. -@par <code>--with-python-prefix</code> -Configure WiredTiger to install Python libraries to a non-standard Python -install location. - -@par <code>--with-spinlock[=pthread, pthread_adaptive, gcc]</code> +@par <code>-DSPINLOCK_TYPE[=pthread, pthread_adaptive, gcc]</code> Configure WiredTiger to use a specific mutex type for serialization; options are \c pthread (the default, which configures WiredTiger to use POSIX 1003.1c pthread mutexes), \c pthread_adaptive (which configures @@ -163,43 +140,27 @@ configures WiredTiger to use gcc-based spinlocks). @section posix_compiler Changing compiler or loader options -To change the compiler or loader behavior during the build, use the -<code>CC</code>, <code>CFLAGS</code>, <code>LDFLAGS</code>, or -<code>LIBS</code> environment variables: - -@par \c CC -The compiler. -@par \c CFLAGS -Compiler flags. -@par \c LDFLAGS -Loader flags. -@par \c LIBS -Additional libraries. - -For example, to specify a different compiler: - -@code -env CC=mygcc ./configure -@endcode +By default CMake will use your default system compiler (\c cc). If you want to use a specific toolchain you can pass a toolchain file. We have +provided a toolchain file for both GCC (\c build_cmake/toolchains/gcc.cmake) and Clang (\c build_cmake/toolchains/clang.cmake). To use either +toolchain you can pass the \c -DCMAKE_TOOLCHAIN_FILE= to the CMake configuration step. For example: -By default, WiredTiger builds with the \c -O3 compiler optimization flag -unless the \c --enable-debug configuration option is specified, in which -case the \c -g compiler flag is used instead. For example, to specify -a different level of optimization: +For example, to explicitly specify the GCC toolchain: @code -env CFLAGS=-Os ./configure +cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/gcc.cmake -G Ninja ../. @endcode -To specify a different set of include files: +To explicitly specify the Clang toolchain: @code -env CFLAGS=-I/usr/local/include ./configure +cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/clang.cmake -G Ninja ../. @endcode -To specify an additional library: +By default, WiredTiger builds with the \c -O3 compiler optimization flag +unless manually specified through the \c -DCC_OPTIMIZE_LEVEL configuration option. For example, to specify +a different level of optimization: @code -env LIBS="-lrtf -lmin" LDFLAGS=-L/usr/local/lib ./configure +cmake -DCC_OPTIMIZE_LEVEL=-01 -G Ninja ../. @endcode - */ +*/ diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index cb5f89ab81d..381b0547ac9 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -5,9 +5,13 @@ ActiveState Adler's ASAN Atomicity +autoconf BLOBs BLRrVv +ccache CFLAGS +cmake +CMake COV CPPFLAGS CPUs @@ -31,7 +35,13 @@ Datastore DbCursor DbEnv DbMultiple +DCC +DCMAKE +DENABLE +DHAVE +DSPINLOCK Durations +DWT EAGAIN EB EBUSY @@ -566,6 +576,7 @@ superset svg sys syscalls +SYSV sz tRuE tablename @@ -581,6 +592,7 @@ timestamp timestamps todo toolchain +toolchains touchpad tpc tpcb @@ -631,6 +643,7 @@ wtstats xa xaa xad +xargs xbe xde xdeadbeef diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 189952f0f42..26f572ce517 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -636,6 +636,11 @@ __evict_update_work(WT_SESSION_IMPL *session) } else LF_SET(WT_CACHE_EVICT_NOKEEP); + if (FLD_ISSET(conn->debug_flags, WT_CONN_DEBUG_UPDATE_RESTORE_EVICT)) { + LF_SET(WT_CACHE_EVICT_SCRUB); + LF_CLR(WT_CACHE_EVICT_NOKEEP); + } + /* * With an in-memory cache, we only do dirty eviction in order to scrub pages. */ @@ -844,7 +849,7 @@ __evict_clear_all_walks(WT_SESSION_IMPL *session) conn = S2C(session); TAILQ_FOREACH (dhandle, &conn->dhqh, q) - if (dhandle->type == WT_DHANDLE_TYPE_BTREE) + if (WT_DHANDLE_BTREE(dhandle)) WT_WITH_DHANDLE(session, dhandle, WT_TRET(__evict_clear_walk(session))); return (ret); } @@ -1463,7 +1468,7 @@ retry: break; /* Ignore non-btree handles, or handles that aren't open. */ - if (dhandle->type != WT_DHANDLE_TYPE_BTREE || !F_ISSET(dhandle, WT_DHANDLE_OPEN)) + if (!WT_DHANDLE_BTREE(dhandle) || !F_ISSET(dhandle, WT_DHANDLE_OPEN)) continue; /* Skip files that don't allow eviction. */ @@ -2661,7 +2666,7 @@ __verbose_dump_cache_apply(WT_SESSION_IMPL *session, uint64_t *total_bytesp, break; /* Skip if the tree is marked discarded by another thread. */ - if (dhandle->type != WT_DHANDLE_TYPE_BTREE || !F_ISSET(dhandle, WT_DHANDLE_OPEN) || + if (!WT_DHANDLE_BTREE(dhandle) || !F_ISSET(dhandle, WT_DHANDLE_OPEN) || F_ISSET(dhandle, WT_DHANDLE_DISCARD)) continue; diff --git a/src/third_party/wiredtiger/src/history/hs_cursor.c b/src/third_party/wiredtiger/src/history/hs_cursor.c index 8dfad4cd983..4f613e7d042 100644 --- a/src/third_party/wiredtiger/src/history/hs_cursor.c +++ b/src/third_party/wiredtiger/src/history/hs_cursor.c @@ -9,80 +9,6 @@ #include "wt_internal.h" /* - * __wt_hs_row_search -- - * Search the history store for a given key and position the cursor on it. - */ -int -__wt_hs_row_search(WT_CURSOR_BTREE *hs_cbt, WT_ITEM *srch_key, bool insert) -{ - WT_BTREE *hs_btree; - WT_CURSOR *hs_cursor; - WT_DECL_RET; - WT_SESSION_IMPL *session; - bool leaf_found; -#ifdef HAVE_DIAGNOSTIC - WT_PAGE *page; -#endif - - hs_btree = CUR2BT(hs_cbt); - session = CUR2S(hs_cbt); - hs_cursor = &hs_cbt->iface; - leaf_found = false; - - /* - * Check whether the search key can be find in the provided leaf page, if exists. Otherwise - * perform a full search. - */ - if (hs_cbt->ref != NULL) { -#ifdef HAVE_DIAGNOSTIC - WT_ORDERED_READ(page, hs_cbt->ref->page); -#endif - /* - * The page must be pinned and we should have a hazard pointer on that. Ensure the page is - * not evictable. - */ - WT_ASSERT(session, __wt_hazard_check(session, hs_cbt->ref, NULL) != NULL); - WT_WITH_BTREE(session, hs_btree, - ret = __wt_row_search(hs_cbt, srch_key, insert, hs_cbt->ref, false, &leaf_found)); - WT_RET(ret); - - /* - * Only use the pinned page search results if search returns an exact match or a slot other - * than the page's boundary slots, if that's not the case, the record might belong on an - * entirely different page. - */ - if (leaf_found && - (hs_cbt->compare != 0 && - (hs_cbt->slot == 0 || hs_cbt->slot == hs_cbt->ref->page->entries - 1))) - leaf_found = false; - - /* Ensure there is no eviction happened on this page. */ - WT_ASSERT(session, page == hs_cbt->ref->page); - if (!leaf_found) - hs_cursor->reset(hs_cursor); - } - - if (!leaf_found) - WT_WITH_BTREE( - session, hs_btree, ret = __wt_row_search(hs_cbt, srch_key, insert, NULL, false, NULL)); - - if (ret == 0 && !insert) { - WT_ERR(__wt_key_return(hs_cbt)); - WT_ERR(__wt_value_return(hs_cbt, hs_cbt->upd_value)); - } - -#ifdef HAVE_DIAGNOSTIC - WT_TRET(__wt_cursor_key_order_init(hs_cbt)); -#endif - - if (0) { -err: - WT_TRET(__cursor_reset(hs_cbt)); - } - return (ret); -} - -/* * __wt_hs_modify -- * Make an update to the history store. * diff --git a/src/third_party/wiredtiger/src/history/hs_rec.c b/src/third_party/wiredtiger/src/history/hs_rec.c index 6e30d425aec..43fc87989e4 100644 --- a/src/third_party/wiredtiger/src/history/hs_rec.c +++ b/src/third_party/wiredtiger/src/history/hs_rec.c @@ -495,11 +495,10 @@ __wt_hs_insert_updates( * away. */ modify_cnt = 0; - for (; updates.size > 0 && - !(upd->txnid == list->onpage_upd->txnid && - upd->start_ts == list->onpage_upd->start_ts); - tmp = full_value, full_value = prev_full_value, prev_full_value = tmp, - upd = prev_upd) { + for (; updates.size > 0; tmp = full_value, full_value = prev_full_value, + prev_full_value = tmp, upd = prev_upd) { + /* We should never insert the onpage value to the history store. */ + WT_ASSERT(session, upd != list->onpage_upd); WT_ASSERT(session, upd->type == WT_UPDATE_STANDARD || upd->type == WT_UPDATE_MODIFY); tombstone = NULL; @@ -610,6 +609,9 @@ __wt_hs_insert_updates( WT_ASSERT(session, __txn_visible_id(session, upd->txnid)); #endif + /* Clear out the insert success flag prior to our insert attempt. */ + __wt_curhs_clear_insert_success(hs_cursor); + /* * Calculate reverse modify and clear the history store records with timestamps when * inserting the first update. Always write on-disk data store updates to the history @@ -629,20 +631,31 @@ __wt_hs_insert_updates( __wt_calc_modify(session, prev_full_value, full_value, prev_full_value->size / 10, entries, &nentries) == 0) { WT_ERR(__wt_modify_pack(hs_cursor, entries, nentries, &modify_value)); - WT_ERR(__hs_insert_record( - session, hs_cursor, btree, key, WT_UPDATE_MODIFY, modify_value, &tw)); + ret = __hs_insert_record( + session, hs_cursor, btree, key, WT_UPDATE_MODIFY, modify_value, &tw); __wt_scr_free(session, &modify_value); ++modify_cnt; } else { modify_cnt = 0; - WT_ERR(__hs_insert_record( - session, hs_cursor, btree, key, WT_UPDATE_STANDARD, full_value, &tw)); + ret = __hs_insert_record( + session, hs_cursor, btree, key, WT_UPDATE_STANDARD, full_value, &tw); + } + + /* + * Flag the update as now in the history store. + * + * Ensure that we don't use `WT_ERR` for the insertions above. Insertion can return + * non-zero even when the update made it into the history store. In those cases we need + * to write the flag to mark the update as having been written to the history store + * before jumping to the error handling. + */ + if (__wt_curhs_check_insert_success(hs_cursor)) { + F_SET(upd, WT_UPDATE_HS); + if (tombstone != NULL) + F_SET(tombstone, WT_UPDATE_HS); } + WT_ERR(ret); - /* Flag the update as now in the history store. */ - F_SET(upd, WT_UPDATE_HS); - if (tombstone != NULL) - F_SET(tombstone, WT_UPDATE_HS); hs_inserted = true; ++insert_cnt; if (squashed) { @@ -651,11 +664,10 @@ __wt_hs_insert_updates( } } - /* If we squash the onpage value, there may be one or more updates left in the stack. */ - if (updates.size > 0) + /* If we squash the onpage value, we increase the counter here. */ + if (squashed) WT_STAT_CONN_DATA_INCR(session, cache_hs_write_squash); - __wt_update_vector_clear(&updates); /* * In the case that the onpage value is an out of order timestamp update and the update * older than it is a tombstone, it remains in the stack. Clean it up. diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index b8a982e1713..2006be2f9f4 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -51,7 +51,7 @@ struct __wt_extlist { uint64_t bytes; /* Byte count */ uint32_t entries; /* Entry count */ - uint32_t logid; /* Written log ID */ + uint32_t objectid; /* Written object ID */ wt_off_t offset; /* Written extent offset */ uint32_t checksum; /* Written extent checksum */ uint32_t size; /* Written extent size */ @@ -141,7 +141,7 @@ struct __wt_size { struct __wt_block_ckpt { uint8_t version; /* Version */ - uint32_t root_logid; + uint32_t root_objectid; wt_off_t root_offset; /* The root */ uint32_t root_checksum, root_size; @@ -239,17 +239,17 @@ struct __wt_block { /* Configuration information, set when the file is opened. */ uint32_t allocfirst; /* Allocation is first-fit */ uint32_t allocsize; /* Allocation size */ - bool log_structured; /* Write checkpoint as separate files */ + bool has_objects; /* Address cookies contain object id */ size_t os_cache; /* System buffer cache flush max */ size_t os_cache_max; size_t os_cache_dirty_max; u_int block_header; /* Header length */ - /* Log-structured tracking. */ - uint32_t file_flags, logid, max_logid; - WT_FH **lfh; - size_t lfh_alloc; + /* Object file tracking. */ + uint32_t file_flags, objectid, max_objectid; + WT_FH **ofh; + size_t ofh_alloc; /* * There is only a single checkpoint in a file that can be written. The information could diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 5283c46df55..f86adf08d53 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -232,8 +232,8 @@ struct __wt_ovfl_reuse { * We also configure a larger than default internal page size to accommodate for larger history * store keys. We do that to reduce the chances of having to create overflow keys on the page. */ -#ifdef HAVE_BUILTIN_EXTENSION_ZSTD -#define WT_HS_COMPRESSOR "zstd" +#ifdef HAVE_BUILTIN_EXTENSION_SNAPPY +#define WT_HS_COMPRESSOR "snappy" #else #define WT_HS_COMPRESSOR "none" #endif diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index fc40f05e0d5..b13ab2b911e 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -229,6 +229,7 @@ struct __wt_connection_impl { WT_SPINLOCK reconfig_lock; /* Single thread reconfigure */ WT_SPINLOCK schema_lock; /* Schema operation spinlock */ WT_RWLOCK table_lock; /* Table list lock */ + WT_SPINLOCK tiered_lock; /* Tiered work queue spinlock */ WT_SPINLOCK turtle_lock; /* Turtle file spinlock */ WT_RWLOCK dhandle_lock; /* Data handle list lock */ @@ -278,13 +279,15 @@ struct __wt_connection_impl { TAILQ_HEAD(__wt_dhhash, __wt_data_handle) * dhhash; /* Locked: data handle list */ TAILQ_HEAD(__wt_dhandle_qh, __wt_data_handle) dhqh; - /* Locked: LSM handle list. */ - TAILQ_HEAD(__wt_lsm_qh, __wt_lsm_tree) lsmqh; + /* Locked: dynamic library handle list */ + TAILQ_HEAD(__wt_dlh_qh, __wt_dlh) dlhqh; /* Locked: file list */ TAILQ_HEAD(__wt_fhhash, __wt_fh) * fhhash; TAILQ_HEAD(__wt_fh_qh, __wt_fh) fhqh; - /* Locked: library list */ - TAILQ_HEAD(__wt_dlh_qh, __wt_dlh) dlhqh; + /* Locked: LSM handle list. */ + TAILQ_HEAD(__wt_lsm_qh, __wt_lsm_tree) lsmqh; + /* Locked: Tiered system work queue. */ + TAILQ_HEAD(__wt_tiered_qh, __wt_tiered_work_unit) tieredqh; WT_SPINLOCK block_lock; /* Locked: block manager list */ TAILQ_HEAD(__wt_blockhash, __wt_block) * blockhash; @@ -413,6 +416,7 @@ struct __wt_connection_impl { const char *stat_stamp; /* Statistics log entry timestamp */ uint64_t stat_usecs; /* Statistics log period */ + uint64_t tiered_retention; /* Earliest time to check to remove local overlap copies */ WT_SESSION_IMPL *tiered_session; /* Tiered thread session */ wt_thread_t tiered_tid; /* Tiered thread */ bool tiered_tid_set; /* Tiered thread set */ @@ -539,6 +543,7 @@ struct __wt_connection_impl { #define WT_CONN_DEBUG_CURSOR_COPY 0x04u #define WT_CONN_DEBUG_REALLOC_EXACT 0x08u #define WT_CONN_DEBUG_SLOW_CKPT 0x10u +#define WT_CONN_DEBUG_UPDATE_RESTORE_EVICT 0x20u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t debug_flags; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 82023a1573e..ffe960de07f 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -290,6 +290,8 @@ struct __wt_cursor_hs { uint32_t btree_id; WT_ITEM *datastore_key; + bool insert_success; + /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_HS_CUR_BTREE_ID_SET 0x1u #define WT_HS_CUR_COUNTER_SET 0x2u diff --git a/src/third_party/wiredtiger/src/include/cursor_inline.h b/src/third_party/wiredtiger/src/include/cursor_inline.h index 8325e0ad8e6..49179efd0eb 100644 --- a/src/third_party/wiredtiger/src/include/cursor_inline.h +++ b/src/third_party/wiredtiger/src/include/cursor_inline.h @@ -95,11 +95,11 @@ __cursor_checkvalue(WT_CURSOR *cursor) } /* - * __cursor_localkey -- + * __wt_cursor_localkey -- * If the key points into the tree, get a local copy. */ static inline int -__cursor_localkey(WT_CURSOR *cursor) +__wt_cursor_localkey(WT_CURSOR *cursor) { if (F_ISSET(cursor, WT_CURSTD_KEY_INT)) { if (!WT_DATA_IN_ITEM(&cursor->key)) @@ -136,7 +136,7 @@ __cursor_localvalue(WT_CURSOR *cursor) static inline int __cursor_needkey(WT_CURSOR *cursor) { - WT_RET(__cursor_localkey(cursor)); + WT_RET(__wt_cursor_localkey(cursor)); return (__cursor_checkkey(cursor)); } @@ -352,10 +352,14 @@ __wt_cursor_dhandle_decr_use(WT_SESSION_IMPL *session) dhandle = session->dhandle; - /* If we close a handle with a time of death set, clear it. */ + /* + * If we close a handle with a time of death set, clear it. The ordering is important: after + * decrementing the use count, there's a chance that the data handle can be freed. + */ WT_ASSERT(session, dhandle->session_inuse > 0); - if (__wt_atomic_subi32(&dhandle->session_inuse, 1) == 0 && dhandle->timeofdeath != 0) + if (dhandle->timeofdeath != 0 && dhandle->session_inuse == 1) dhandle->timeofdeath = 0; + (void)__wt_atomic_subi32(&dhandle->session_inuse, 1); } /* @@ -404,11 +408,11 @@ __cursor_kv_return(WT_CURSOR_BTREE *cbt, WT_UPDATE_VALUE *upd_value) } /* - * __cursor_func_init -- + * __wt_cursor_func_init -- * Cursor call setup. */ static inline int -__cursor_func_init(WT_CURSOR_BTREE *cbt, bool reenter) +__wt_cursor_func_init(WT_CURSOR_BTREE *cbt, bool reenter) { WT_SESSION_IMPL *session; diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 1ca46550587..01cfeffce6e 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -64,12 +64,15 @@ struct __wt_data_handle { TAILQ_ENTRY(__wt_data_handle) q; TAILQ_ENTRY(__wt_data_handle) hashq; - const char *name; /* Object name as a URI */ - uint64_t name_hash; /* Hash of name */ - const char *checkpoint; /* Checkpoint name (or NULL) */ - const char **cfg; /* Configuration information */ - const char *meta_base; /* Base metadata configuration */ - + const char *name; /* Object name as a URI */ + uint64_t name_hash; /* Hash of name */ + const char *checkpoint; /* Checkpoint name (or NULL) */ + const char **cfg; /* Configuration information */ + const char *meta_base; /* Base metadata configuration */ + size_t meta_base_length; /* Base metadata length */ +#ifdef HAVE_DIAGNOSTIC + const char *orig_meta_base; /* Copy of the base metadata configuration */ +#endif /* * Sessions holding a connection's data handle will have a non-zero reference count; sessions * using a connection's data handle will have a non-zero in-use count. Instances of cached @@ -91,6 +94,9 @@ struct __wt_data_handle { WT_DHANDLE_TYPE_TIERED_TREE } type; + /* This will include the tiered type soon. */ +#define WT_DHANDLE_BTREE(dhandle) ((dhandle)->type == WT_DHANDLE_TYPE_BTREE) + bool compact_skip; /* If the handle failed to compact */ /* diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 15fbcdb4a74..e89dd2c943e 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -14,6 +14,8 @@ extern bool __wt_cell_type_check(uint8_t cell_type, uint8_t dsk_type) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_checksum_alt_match(const void *chunk, size_t len, uint32_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_curhs_check_insert_success(WT_CURSOR *cursor) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_delete_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool visible_all) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_evict_thread_chk(WT_SESSION_IMPL *session) @@ -87,11 +89,12 @@ extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, co size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t logid, wt_off_t offset, - uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, + wt_off_t offset, uint32_t size, uint32_t checksum) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_alloc(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t *offp, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *logidp, +extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, @@ -153,7 +156,7 @@ extern int __wt_block_extlist_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_extlist_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, WT_EXTLIST *additional) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, WT_FH **fhp) +extern int __wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -175,7 +178,7 @@ extern int __wt_block_map(WT_SESSION_IMPL *session, WT_BLOCK *block, void *mappe extern int __wt_block_misplaced(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *list, wt_off_t offset, uint32_t size, bool live, const char *func, int line) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_off_free(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, +extern int __wt_block_off_free(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, wt_off_t offset, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_off_remove_overlap(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t off, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -183,7 +186,7 @@ extern int __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const bool forced_salvage, bool readonly, uint32_t allocsize, WT_BLOCK **blockp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, - uint32_t logid, wt_off_t offset, uint32_t size, uint32_t checksum) + uint32_t objectid, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_read_off_blind(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset, uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -217,7 +220,7 @@ extern int __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM * size_t *addr_sizep, bool data_checksum, bool checkpoint_io) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, - uint32_t *logidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, + uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write_size(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -326,7 +329,7 @@ extern int __wt_bulk_insert_var(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) +extern int __wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -602,9 +605,6 @@ extern int __wt_curtable_get_value(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_curtiered_close(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_curtiered_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, - const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_addr(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_debug_addr_print(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) @@ -779,8 +779,6 @@ extern int __wt_hs_modify(WT_CURSOR_BTREE *hs_cbt, WT_UPDATE *hs_upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_open(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_hs_row_search(WT_CURSOR_BTREE *hs_cbt, WT_ITEM *srch_key, bool insert) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_verify(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_verify_one(WT_SESSION_IMPL *session) @@ -1454,6 +1452,10 @@ extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP * extern int __wt_thread_group_resize(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t new_min, uint32_t new_max, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tier_do_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *local_uri, + const char *obj_uri) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tier_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint64_t id) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_bucket_config(WT_SESSION_IMPL *session, const char *cfg[], WT_BUCKET_STORAGE **bstoragep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_close(WT_SESSION_IMPL *session, WT_TIERED *tiered) @@ -1464,7 +1466,13 @@ extern int __wt_tiered_name(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, u uint32_t flags, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig) +extern int __wt_tiered_put_drop_local(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint64_t id) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_put_drop_shared(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint64_t id) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_put_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_storage_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -1682,6 +1690,7 @@ extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); +extern void __wt_curhs_clear_insert_success(WT_CURSOR *cursor); extern void __wt_cursor_close(WT_CURSOR *cursor); extern void __wt_cursor_get_hash( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *to_dup, uint64_t *hash_value); @@ -1809,6 +1818,13 @@ extern void __wt_stat_session_init_single(WT_SESSION_STATS *stats); extern void __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool is_locked); extern void __wt_thread_group_stop_one(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group); +extern void __wt_tiered_get_drop_local( + WT_SESSION_IMPL *session, uint64_t now, WT_TIERED_WORK_UNIT **entryp); +extern void __wt_tiered_get_drop_shared(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT **entryp); +extern void __wt_tiered_get_flush(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT **entryp); +extern void __wt_tiered_pop_work( + WT_SESSION_IMPL *session, uint32_t type, uint64_t maxval, WT_TIERED_WORK_UNIT **entryp); +extern void __wt_tiered_push_work(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT *entry); extern void __wt_timestamp_to_hex_string(wt_timestamp_t ts, char *hex_timestamp); extern void __wt_txn_bump_snapshot(WT_SESSION_IMPL *session); extern void __wt_txn_clear_durable_timestamp(WT_SESSION_IMPL *session); @@ -1966,6 +1982,10 @@ static inline int __wt_compare_skip(WT_SESSION_IMPL *session, WT_COLLATOR *colla WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_curindex_get_valuev(WT_CURSOR *cursor, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_cursor_func_init(WT_CURSOR_BTREE *cbt, bool reenter) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_cursor_localkey(WT_CURSOR *cursor) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_curtable_get_valuev(WT_CURSOR *cursor, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_dsk_cell_data_ref(WT_SESSION_IMPL *session, int page_type, void *unpack_arg, diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 540c79187b4..3f4bd5150e3 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -777,6 +777,8 @@ struct __wt_connection_stats { int64_t rec_time_window_durable_stop_ts; int64_t rec_time_window_stop_ts; int64_t rec_time_window_stop_txn; + int64_t tiered_work_units_dequeued; + int64_t tiered_work_units_created; int64_t tiered_retention; int64_t tiered_object_size; int64_t txn_read_race_prepare_update; @@ -996,6 +998,8 @@ struct __wt_dsrc_stats { int64_t rec_time_window_durable_stop_ts; int64_t rec_time_window_stop_ts; int64_t rec_time_window_stop_txn; + int64_t tiered_work_units_dequeued; + int64_t tiered_work_units_created; int64_t tiered_retention; int64_t tiered_object_size; int64_t txn_read_race_prepare_update; diff --git a/src/third_party/wiredtiger/src/include/tiered.h b/src/third_party/wiredtiger/src/include/tiered.h index 6cd9162e1c4..8fa2fec35a7 100644 --- a/src/third_party/wiredtiger/src/include/tiered.h +++ b/src/third_party/wiredtiger/src/include/tiered.h @@ -27,28 +27,6 @@ struct __wt_tiered_manager { }; /* - * WT_CURSOR_TIERED -- - * A tiered cursor. - */ -struct __wt_cursor_tiered { - WT_CURSOR iface; - - WT_TIERED *tiered; - - WT_CURSOR **cursors; - WT_CURSOR *current; /* The current cursor for iteration */ - WT_CURSOR *primary; /* The current primary */ - -/* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_CURTIERED_ACTIVE 0x1u /* Incremented the session count */ -#define WT_CURTIERED_ITERATE_NEXT 0x2u /* Forward iteration */ -#define WT_CURTIERED_ITERATE_PREV 0x4u /* Backward iteration */ -#define WT_CURTIERED_MULTIPLE 0x8u /* Multiple cursors have values */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ - uint32_t flags; -}; - -/* * Define the maximum number of tiers for convenience. We expect at most two initially. This can * change if more are needed. It is easier to have the array statically allocated initially than * worrying about the memory management. For now also assign types to slots. Local files in slot 0. @@ -69,6 +47,32 @@ struct __wt_cursor_tiered { /* AUTOMATIC FLAG VALUE GENERATION STOP */ /* + * Different types of work units for tiered trees. + */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TIERED_WORK_DROP_LOCAL 0x1u /* Drop object from local storage. */ +#define WT_TIERED_WORK_DROP_SHARED 0x2u /* Drop object from tier. */ +#define WT_TIERED_WORK_FLUSH 0x4u /* Flush object to tier. */ +/* AUTOMATIC FLAG VALUE GENERATION STOP */ + +/* + * WT_TIERED_WORK_UNIT -- + * A definition of maintenance that a tiered tree needs done. + */ +struct __wt_tiered_work_unit { + TAILQ_ENTRY(__wt_tiered_work_unit) q; /* Worker unit queue */ + uint32_t type; /* Type of operation */ + uint64_t op_val; /* A value for the operation */ + WT_TIERED *tiered; /* Tiered tree */ + uint64_t id; /* Id of the object */ +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TIERED_WORK_FORCE 0x1u /* Force operation */ +#define WT_TIERED_WORK_FREE 0x2u /* Free data after operation */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t flags; /* Flags for operation */ +}; + +/* * WT_TIERED_TIERS -- * Information we need to keep about each tier such as its data handle and name. * We define operations that each tier can accept. The local tier should be able to accept diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index c21c3dac748..d0a4c0ad38e 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1068,10 +1068,8 @@ struct __wt_session { * empty.} * @config{block_allocation, configure block allocation. Permitted values are \c "best" or * \c "first"; the \c "best" configuration uses a best-fit algorithm\, the \c "first" - * configuration uses a first-available algorithm during block allocation\, the \c - * "log-structure" configuration allocates a new file for each checkpoint., a string\, - * chosen from the following options: \c "best"\, \c "first"\, \c "log-structured"; default - * \c best.} + * configuration uses a first-available algorithm during block allocation., a string\, + * chosen from the following options: \c "best"\, \c "first"; default \c best.} * @config{block_compressor, configure a compressor for file blocks. Permitted values are * \c "none" or custom compression engine name created with WT_CONNECTION::add_compressor. * If WiredTiger has builtin support for \c "lz4"\, \c "snappy"\, \c "zlib" or \c "zstd" @@ -2076,6 +2074,8 @@ struct __wt_connection { * operations\, even operations for tables with logging turned off. This setting introduces * a log format change that may break older versions of WiredTiger. These operations are * informational and skipped in recovery., a boolean flag; default \c false.} + * @config{ update_restore_evict, if true\, control all dirty page + * evictions through forcing update restore eviction., a boolean flag; default \c false.} * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; default empty.} * @config{eviction = (, eviction configuration options., a set of related configuration @@ -2740,6 +2740,9 @@ struct __wt_connection { * tables with logging turned off. This setting introduces a log format change that may break older * versions of WiredTiger. These operations are informational and skipped in recovery., a boolean * flag; default \c false.} + * @config{ update_restore_evict, if true\, control + * all dirty page evictions through forcing update restore eviction., a boolean flag; default \c + * false.} * @config{ ),,} * @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c FILE_FLAG_NO_BUFFERING on Windows to * access files. Options are given as a list\, such as <code>"direct_io=[data]"</code>. Configuring @@ -6064,35 +6067,39 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1462 /*! reconciliation: records written including a stop transaction ID */ #define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1463 +/*! session: tiered operations dequeued and processed */ +#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1464 +/*! session: tiered operations scheduled */ +#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1465 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_CONN_TIERED_RETENTION 1464 +#define WT_STAT_CONN_TIERED_RETENTION 1466 /*! session: tiered storage object size */ -#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1465 +#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1467 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1466 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1468 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1467 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1469 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1468 +#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1470 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1469 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1471 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1470 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1472 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1471 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1473 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1472 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1474 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1473 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1475 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1474 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1476 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1475 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1477 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1476 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1478 /*! * @} @@ -6688,35 +6695,39 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2197 /*! reconciliation: records written including a stop transaction ID */ #define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2198 +/*! session: tiered operations dequeued and processed */ +#define WT_STAT_DSRC_TIERED_WORK_UNITS_DEQUEUED 2199 +/*! session: tiered operations scheduled */ +#define WT_STAT_DSRC_TIERED_WORK_UNITS_CREATED 2200 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_DSRC_TIERED_RETENTION 2199 +#define WT_STAT_DSRC_TIERED_RETENTION 2201 /*! session: tiered storage object size */ -#define WT_STAT_DSRC_TIERED_OBJECT_SIZE 2200 +#define WT_STAT_DSRC_TIERED_OBJECT_SIZE 2202 /*! transaction: race to read prepared update retry */ -#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2201 +#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2203 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2202 +#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2204 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_DSRC_TXN_RTS_INCONSISTENT_CKPT 2203 +#define WT_STAT_DSRC_TXN_RTS_INCONSISTENT_CKPT 2205 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2204 +#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2206 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2205 +#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2207 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2206 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2208 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2207 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2209 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2208 +#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2210 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2209 +#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2211 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2210 +#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2212 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2211 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2213 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index b29575e0628..4f896a73525 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -167,8 +167,6 @@ struct __wt_cursor_stat; typedef struct __wt_cursor_stat WT_CURSOR_STAT; struct __wt_cursor_table; typedef struct __wt_cursor_table WT_CURSOR_TABLE; -struct __wt_cursor_tiered; -typedef struct __wt_cursor_tiered WT_CURSOR_TIERED; struct __wt_data_handle; typedef struct __wt_data_handle WT_DATA_HANDLE; struct __wt_data_handle_cache; @@ -327,6 +325,8 @@ struct __wt_tiered_tiers; typedef struct __wt_tiered_tiers WT_TIERED_TIERS; struct __wt_tiered_tree; typedef struct __wt_tiered_tree WT_TIERED_TREE; +struct __wt_tiered_work_unit; +typedef struct __wt_tiered_work_unit WT_TIERED_WORK_UNIT; struct __wt_time_aggregate; typedef struct __wt_time_aggregate WT_TIME_AGGREGATE; struct __wt_time_window; diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index b5d0ae0b7a1..d4c8f316e5f 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -193,10 +193,12 @@ __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) static int __ckpt_set(WT_SESSION_IMPL *session, const char *fname, const char *v, bool use_base) { + WT_DATA_HANDLE *dhandle; WT_DECL_ITEM(tmp); WT_DECL_RET; + size_t meta_base_length; char *config, *newcfg; - const char *cfg[3], *str; + const char *cfg[3], *meta_base, *str; /* * If the caller knows we're on a path like checkpoints where we have a valid checkpoint and @@ -205,12 +207,35 @@ __ckpt_set(WT_SESSION_IMPL *session, const char *fname, const char *v, bool use_ * use the slower path through configuration parsing functions. */ config = newcfg = NULL; + dhandle = session->dhandle; str = v == NULL ? "checkpoint=(),checkpoint_backup_info=(),checkpoint_lsn=" : v; - if (use_base && session->dhandle != NULL) { + if (use_base && dhandle != NULL) { WT_ERR(__wt_scr_alloc(session, 0, &tmp)); - WT_ASSERT(session, strcmp(session->dhandle->name, fname) == 0); + WT_ASSERT(session, strcmp(dhandle->name, fname) == 0); + + /* Check the metadata is not corrupted. */ + meta_base = dhandle->meta_base; + meta_base_length = strlen(meta_base); + if (dhandle->meta_base_length != meta_base_length) + WT_ERR_PANIC(session, WT_PANIC, + "Corrupted metadata. The original metadata length was %lu while the new one is %lu.", + dhandle->meta_base_length, meta_base_length); +#ifdef HAVE_DIAGNOSTIC + if (!WT_STREQ(dhandle->orig_meta_base, meta_base)) + WT_ERR_PANIC(session, WT_PANIC, + "Corrupted metadata. The original metadata length was %lu while the new one is %lu. " + "The original metadata inserted was %s and the current " + "metadata is now %s.", + dhandle->meta_base_length, meta_base_length, dhandle->orig_meta_base, meta_base); +#endif + /* Concatenate the metadata base string with the checkpoint string. */ - WT_ERR(__wt_buf_fmt(session, tmp, "%s,%s", session->dhandle->meta_base, str)); + WT_ERR(__wt_buf_fmt(session, tmp, "%s,%s", meta_base, str)); + /* + * Check the new metadata length is at least as long as the original metadata string with + * the checkpoint base stripped out. + */ + WT_ASSERT(session, tmp->size >= dhandle->meta_base_length); WT_ERR(__wt_metadata_update(session, fname, tmp->mem)); } else { /* Retrieve the metadata for this file. */ diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index d1e4d909b50..ef5c3246e30 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -255,7 +255,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v upd_memsize = 0; max_ts = WT_TS_NONE; max_txn = WT_TXN_NONE; - has_newer_updates = upd_saved = false; + has_newer_updates = supd_restore = upd_saved = false; is_hs_page = F_ISSET(session->dhandle, WT_DHANDLE_HS); session_txnid = WT_SESSION_TXN_SHARED(session)->id; @@ -615,8 +615,6 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v supd_restore = F_ISSET(r, WT_REC_EVICT) && (has_newer_updates || F_ISSET(S2C(session), WT_CONN_IN_MEMORY) || page->type == WT_PAGE_COL_FIX); - if (supd_restore) - r->cache_write_restore = true; WT_ERR(__rec_update_save(session, r, ins, ripcip, upd_select->upd != NULL && upd_select->upd->type == WT_UPDATE_TOMBSTONE ? NULL : upd_select->upd, @@ -634,6 +632,13 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v } /* + * Set statistics for update restore evictions. Update restore eviction debug mode forces update + * restores to both committed or uncommitted changes. + */ + if (supd_restore || F_ISSET(r, WT_REC_SCRUB)) + r->cache_write_restore = true; + + /* * Paranoia: check that we didn't choose an update that has since been rolled back. */ WT_ASSERT(session, upd_select->upd == NULL || upd_select->upd->txnid != WT_TXN_ABORTED); diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index 930fdedba8b..3bc35ddcd89 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -79,6 +79,19 @@ __alter_file(WT_SESSION_IMPL *session, const char *newcfg[]) } /* + * __alter_object -- + * Alter a tiered object. There are no object dhandles. + */ +static int +__alter_object(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) +{ + if (!WT_PREFIX_MATCH(uri, "object:")) + return (__wt_unexpected_object_type(session, uri, "object:")); + + return (__alter_apply(session, uri, newcfg, WT_CONFIG_BASE(session, object_meta))); +} + +/* * __alter_tree -- * Alter an index or colgroup reference. */ @@ -218,6 +231,8 @@ __schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) return (__alter_tree(session, uri, newcfg)); if (WT_PREFIX_MATCH(uri, "lsm:")) return (__wt_lsm_tree_worker(session, uri, __alter_file, NULL, newcfg, flags)); + if (WT_PREFIX_MATCH(uri, "object:")) + return (__alter_object(session, uri, newcfg)); if (WT_PREFIX_MATCH(uri, "table:")) return (__alter_table(session, uri, newcfg, exclusive_refreshed)); if (WT_PREFIX_MATCH(uri, "tiered:")) diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index a6796946c99..d47cfa46af8 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -452,7 +452,7 @@ __session_open_cursor_int(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR * if (WT_PREFIX_MATCH(uri, "table:")) WT_RET(__wt_curtable_open(session, uri, owner, cfg, cursorp)); if (WT_PREFIX_MATCH(uri, "tiered:")) - WT_RET(__wt_curtiered_open(session, uri, owner, cfg, cursorp)); + WT_RET(__wt_curfile_open(session, uri, owner, cfg, cursorp)); break; case 'c': if (WT_PREFIX_MATCH(uri, "colgroup:")) { diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 98e58f56aea..8172fc8841d 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -380,7 +380,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) { dhandle = dhandle_cache->dhandle; empty_btree = false; - if (dhandle->type == WT_DHANDLE_TYPE_BTREE) + if (WT_DHANDLE_BTREE(dhandle)) WT_WITH_DHANDLE( session, dhandle, empty_btree = (__wt_btree_bytes_evictable(session) == 0)); diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index f4c20e02746..81212ff65da 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -573,8 +573,9 @@ __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_ATTR if (WT_PREFIX_MATCH(uri, "backup:") || WT_PREFIX_MATCH(uri, "colgroup:") || WT_PREFIX_MATCH(uri, "config:") || WT_PREFIX_MATCH(uri, "file:") || WT_PREFIX_MATCH(uri, "index:") || WT_PREFIX_MATCH(uri, "log:") || - WT_PREFIX_MATCH(uri, "lsm:") || WT_PREFIX_MATCH(uri, "statistics:") || - WT_PREFIX_MATCH(uri, "table:") || WT_PREFIX_MATCH(uri, "tiered:")) + WT_PREFIX_MATCH(uri, "lsm:") || WT_PREFIX_MATCH(uri, "object:") || + WT_PREFIX_MATCH(uri, "statistics:") || WT_PREFIX_MATCH(uri, "table:") || + WT_PREFIX_MATCH(uri, "tiered:")) return (__wt_object_unsupported(session, uri)); WT_RET_MSG(session, ENOTSUP, "unknown object type: %s", uri); diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index fb9f7870a54..647d87bf9fb 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -207,6 +207,8 @@ static const char *const __stats_dsrc_desc[] = { "reconciliation: records written including a stop durable timestamp", "reconciliation: records written including a stop timestamp", "reconciliation: records written including a stop transaction ID", + "session: tiered operations dequeued and processed", + "session: tiered operations scheduled", "session: tiered storage local retention time (secs)", "session: tiered storage object size", "transaction: race to read prepared update retry", @@ -460,6 +462,8 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->rec_time_window_durable_stop_ts = 0; stats->rec_time_window_stop_ts = 0; stats->rec_time_window_stop_txn = 0; + stats->tiered_work_units_dequeued = 0; + stats->tiered_work_units_created = 0; /* not clearing tiered_retention */ /* not clearing tiered_object_size */ stats->txn_read_race_prepare_update = 0; @@ -699,6 +703,8 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->rec_time_window_durable_stop_ts += from->rec_time_window_durable_stop_ts; to->rec_time_window_stop_ts += from->rec_time_window_stop_ts; to->rec_time_window_stop_txn += from->rec_time_window_stop_txn; + to->tiered_work_units_dequeued += from->tiered_work_units_dequeued; + to->tiered_work_units_created += from->tiered_work_units_created; to->tiered_retention += from->tiered_retention; to->tiered_object_size += from->tiered_object_size; to->txn_read_race_prepare_update += from->txn_read_race_prepare_update; @@ -945,6 +951,8 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->rec_time_window_durable_stop_ts += WT_STAT_READ(from, rec_time_window_durable_stop_ts); to->rec_time_window_stop_ts += WT_STAT_READ(from, rec_time_window_stop_ts); to->rec_time_window_stop_txn += WT_STAT_READ(from, rec_time_window_stop_txn); + to->tiered_work_units_dequeued += WT_STAT_READ(from, tiered_work_units_dequeued); + to->tiered_work_units_created += WT_STAT_READ(from, tiered_work_units_created); to->tiered_retention += WT_STAT_READ(from, tiered_retention); to->tiered_object_size += WT_STAT_READ(from, tiered_object_size); to->txn_read_race_prepare_update += WT_STAT_READ(from, txn_read_race_prepare_update); @@ -1436,6 +1444,8 @@ static const char *const __stats_connection_desc[] = { "reconciliation: records written including a stop durable timestamp", "reconciliation: records written including a stop timestamp", "reconciliation: records written including a stop transaction ID", + "session: tiered operations dequeued and processed", + "session: tiered operations scheduled", "session: tiered storage local retention time (secs)", "session: tiered storage object size", "transaction: race to read prepared update retry", @@ -1954,6 +1964,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->rec_time_window_durable_stop_ts = 0; stats->rec_time_window_stop_ts = 0; stats->rec_time_window_stop_txn = 0; + stats->tiered_work_units_dequeued = 0; + stats->tiered_work_units_created = 0; /* not clearing tiered_retention */ /* not clearing tiered_object_size */ stats->txn_read_race_prepare_update = 0; @@ -2483,6 +2495,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->rec_time_window_durable_stop_ts += WT_STAT_READ(from, rec_time_window_durable_stop_ts); to->rec_time_window_stop_ts += WT_STAT_READ(from, rec_time_window_stop_ts); to->rec_time_window_stop_txn += WT_STAT_READ(from, rec_time_window_stop_txn); + to->tiered_work_units_dequeued += WT_STAT_READ(from, tiered_work_units_dequeued); + to->tiered_work_units_created += WT_STAT_READ(from, tiered_work_units_created); to->tiered_retention += WT_STAT_READ(from, tiered_retention); to->tiered_object_size += WT_STAT_READ(from, tiered_object_size); to->txn_read_race_prepare_update += WT_STAT_READ(from, txn_read_race_prepare_update); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_config.c b/src/third_party/wiredtiger/src/tiered/tiered_config.c index 6971ec4b7b5..8c0ec27333e 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_config.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_config.c @@ -42,6 +42,8 @@ __tiered_common_config(WT_SESSION_IMPL *session, const char **cfg, WT_BUCKET_STO { WT_CONFIG_ITEM cval; + if (bstorage == NULL) + return (0); WT_RET(__wt_config_gets(session, cfg, "tiered_storage.local_retention", &cval)); bstorage->retain_secs = (uint64_t)cval.val; @@ -153,6 +155,8 @@ __wt_tiered_conn_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfi if (!reconfig) WT_RET(__wt_tiered_bucket_config(session, cfg, &conn->bstorage)); + else + WT_ERR(__tiered_common_config(session, cfg, conn->bstorage)); /* If the connection is not set up for tiered storage there is nothing more to do. */ if (conn->bstorage == NULL) @@ -161,13 +165,7 @@ __wt_tiered_conn_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfi __wt_verbose( session, WT_VERB_TIERED, "TIERED_CONFIG: prefix %s", conn->bstorage->bucket_prefix); - /* - * If reconfiguring, see if the other settings have changed on the system bucket storage. - */ WT_ASSERT(session, conn->bstorage != NULL); - if (reconfig) - WT_ERR(__tiered_common_config(session, cfg, conn->bstorage)); - WT_STAT_CONN_SET(session, tiered_object_size, conn->bstorage->object_size); WT_STAT_CONN_SET(session, tiered_retention, conn->bstorage->retain_secs); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_cursor.c b/src/third_party/wiredtiger/src/tiered/tiered_cursor.c deleted file mode 100644 index c913f9b33ca..00000000000 --- a/src/third_party/wiredtiger/src/tiered/tiered_cursor.c +++ /dev/null @@ -1,1228 +0,0 @@ -/*- - * Copyright (c) 2014-present MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -#define WT_FORALL_CURSORS(curtiered, c, i) \ - for ((i) = 0; i < WT_TIERED_MAX_TIERS;) \ - if (((c) = (curtiered)->cursors[(i)++]) != NULL) - -#define WT_TIERED_CURCMP(s, tiered, c1, c2, cmp) \ - __wt_compare(s, (tiered)->collator, &(c1)->key, &(c2)->key, &(cmp)) - -/* - * __curtiered_open_cursors -- - * Open cursors for the current set of files. - */ -static int -__curtiered_open_cursors(WT_CURSOR_TIERED *curtiered) -{ - WT_CURSOR *cursor; - WT_DATA_HANDLE *dhandle; - WT_DECL_RET; - WT_SESSION_IMPL *session; - WT_TIERED *tiered; - u_int i; - - cursor = &curtiered->iface; - session = CUR2S(curtiered); - dhandle = NULL; - tiered = curtiered->tiered; - - /* - * If the key is pointing to memory that is pinned by a tier cursor, take a copy before closing - * cursors. - */ - if (F_ISSET(cursor, WT_CURSTD_KEY_INT)) - WT_ERR(__cursor_needkey(cursor)); - - F_CLR(curtiered, WT_CURTIERED_ITERATE_NEXT | WT_CURTIERED_ITERATE_PREV); - - WT_ASSERT(session, curtiered->cursors == NULL); - WT_ERR(__wt_calloc_def(session, WT_TIERED_MAX_TIERS, &curtiered->cursors)); - - /* Open the cursors for tiers that have changed. */ - __wt_verbose(session, WT_VERB_TIERED, - "tiered opening cursor session(%p):tiered cursor(%p), tiers: %d", (void *)session, - (void *)curtiered, (int)WT_TIERED_MAX_TIERS); - for (i = 0; i < WT_TIERED_MAX_TIERS; i++) { - dhandle = tiered->tiers[i].tier; - if (dhandle == NULL) - continue; - - /* - * Read from the checkpoint if the file has been written. Once all cursors switch, the - * in-memory tree can be evicted. - */ - WT_ASSERT(session, curtiered->cursors[i] == NULL); - WT_ERR(__wt_open_cursor(session, dhandle->name, cursor, NULL, &curtiered->cursors[i])); - - /* Child cursors always use overwrite and raw mode. */ - F_SET(curtiered->cursors[i], WT_CURSTD_OVERWRITE | WT_CURSTD_RAW); - } - -err: - return (ret); -} - -/* - * __curtiered_close_cursors -- - * Close any btree cursors that are not needed. - */ -static int -__curtiered_close_cursors(WT_SESSION_IMPL *session, WT_CURSOR_TIERED *curtiered) -{ - WT_CURSOR *c; - u_int i; - - __wt_verbose(session, WT_VERB_TIERED, "tiered close cursors session(%p):tiered cursor(%p)", - (void *)session, (void *)curtiered); - - if (curtiered->cursors == NULL) - return (0); - - /* Walk the cursors, closing them. */ - for (i = 0; i < WT_TIERED_MAX_TIERS; i++) { - if ((c = (curtiered)->cursors[i]) != NULL) { - curtiered->cursors[i] = NULL; - WT_RET(c->close(c)); - } - } - - __wt_free(session, curtiered->cursors); - return (0); -} - -/* - * __curtiered_reset_cursors -- - * Reset any positioned tier cursors. If the skip parameter is non-NULL, that cursor is about to - * be used, so there is no need to reset it. - */ -static int -__curtiered_reset_cursors(WT_CURSOR_TIERED *curtiered, WT_CURSOR *skip) -{ - WT_CURSOR *c; - WT_DECL_RET; - u_int i; - - /* Fast path if the cursor is not positioned. */ - if ((curtiered->current == NULL || curtiered->current == skip) && - !F_ISSET(curtiered, WT_CURTIERED_ITERATE_NEXT | WT_CURTIERED_ITERATE_PREV)) - return (0); - - WT_FORALL_CURSORS(curtiered, c, i) - { - if (c == skip) - continue; - if (F_ISSET(c, WT_CURSTD_KEY_INT)) - WT_TRET(c->reset(c)); - } - - curtiered->current = NULL; - F_CLR(curtiered, WT_CURTIERED_ITERATE_NEXT | WT_CURTIERED_ITERATE_PREV); - - return (ret); -} - -/* - * __curtiered_enter -- - * Start an operation on a tiered cursor. - */ -static inline int -__curtiered_enter(WT_CURSOR_TIERED *curtiered, bool reset) -{ - WT_SESSION_IMPL *session; - - session = CUR2S(curtiered); - - if (curtiered->cursors == NULL) - WT_RET(__curtiered_open_cursors(curtiered)); - - if (reset) { - WT_ASSERT(session, !F_ISSET(&curtiered->iface, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT)); - WT_RET(__curtiered_reset_cursors(curtiered, NULL)); - } - - if (!F_ISSET(curtiered, WT_CURTIERED_ACTIVE)) { - /* - * Opening this tiered cursor has opened a number of other cursors, ensure we don't mistake - * this as the first cursor in a session. - */ - ++session->ncursors; - WT_RET(__cursor_enter(session)); - F_SET(curtiered, WT_CURTIERED_ACTIVE); - } - - return (0); -} -/* - * __curtiered_leave -- - * Finish an operation on a tiered cursor. - */ -static void -__curtiered_leave(WT_CURSOR_TIERED *curtiered) -{ - WT_SESSION_IMPL *session; - - session = CUR2S(curtiered); - - if (F_ISSET(curtiered, WT_CURTIERED_ACTIVE)) { - --session->ncursors; - __cursor_leave(session); - F_CLR(curtiered, WT_CURTIERED_ACTIVE); - } -} - -/* - * We need a tombstone to mark deleted records, and we use the special value below for that purpose. - * We use two 0x14 (Device Control 4) bytes to minimize the likelihood of colliding with an - * application-chosen encoding byte, if the application uses two leading DC4 byte for some reason, - * we'll do a wasted data copy each time a new value is inserted into the object. - */ -static const WT_ITEM __tombstone = {"\x14\x14", 2, NULL, 0, 0}; - -/* - * __curtiered_deleted -- - * Check whether the current value is a tombstone. - */ -static inline bool -__curtiered_deleted(WT_CURSOR_TIERED *curtiered, const WT_ITEM *item) -{ - WT_UNUSED(curtiered); - return (item->size == __tombstone.size && - memcmp(item->data, __tombstone.data, __tombstone.size) == 0); -} - -/* - * __curtiered_deleted_encode -- - * Encode values that are in the encoded name space. - */ -static inline int -__curtiered_deleted_encode( - WT_SESSION_IMPL *session, const WT_ITEM *value, WT_ITEM *final_value, WT_ITEM **tmpp) -{ - WT_ITEM *tmp; - - /* - * If value requires encoding, get a scratch buffer of the right size and create a copy of the - * data with the first byte of the tombstone appended. - */ - if (value->size >= __tombstone.size && - memcmp(value->data, __tombstone.data, __tombstone.size) == 0) { - WT_RET(__wt_scr_alloc(session, value->size + 1, tmpp)); - tmp = *tmpp; - - memcpy(tmp->mem, value->data, value->size); - memcpy((uint8_t *)tmp->mem + value->size, __tombstone.data, 1); - final_value->data = tmp->mem; - final_value->size = value->size + 1; - } else { - final_value->data = value->data; - final_value->size = value->size; - } - - return (0); -} - -/* - * __curtiered_deleted_decode -- - * Decode values that start with the tombstone. - */ -static inline void -__curtiered_deleted_decode(WT_CURSOR_TIERED *curtiered, WT_ITEM *value) -{ - WT_UNUSED(curtiered); - /* - * Take care with this check: when a tiered cursor is used for a merge, it is valid to return - * the tombstone value. - */ - if (value->size > __tombstone.size && - memcmp(value->data, __tombstone.data, __tombstone.size) == 0) - --value->size; -} - -/* - * __wt_curtiered_close -- - * WT_CURSOR->close method for the tiered cursor type. - */ -int -__wt_curtiered_close(WT_CURSOR *cursor) -{ - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - /* - * Don't use the normal __curtiered_enter path: that is wasted work when closing, and the cursor - * may never have been used. - */ - curtiered = (WT_CURSOR_TIERED *)cursor; - CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, NULL); -err: - WT_TRET(__curtiered_close_cursors(session, curtiered)); - - /* In case we were somehow left positioned, clear that. */ - __curtiered_leave(curtiered); - - if (curtiered->tiered != NULL) - WT_WITH_DHANDLE(session, (WT_DATA_HANDLE *)curtiered->tiered, - WT_TRET(__wt_session_release_dhandle(session))); - __wt_cursor_close(cursor); - - API_END_RET(session, ret); -} - -/* - * __curtiered_get_current -- - * Find the smallest / largest of the cursors and copy its key/value. - */ -static int -__curtiered_get_current( - WT_SESSION_IMPL *session, WT_CURSOR_TIERED *curtiered, bool smallest, bool *deletedp) -{ - WT_CURSOR *c, *current; - u_int i; - int cmp; - bool multiple; - - current = NULL; - multiple = false; - - WT_FORALL_CURSORS(curtiered, c, i) - { - if (!F_ISSET(c, WT_CURSTD_KEY_INT)) - continue; - if (current == NULL) { - current = c; - continue; - } - WT_RET(WT_TIERED_CURCMP(session, curtiered->tiered, c, current, cmp)); - if (smallest ? cmp < 0 : cmp > 0) { - current = c; - multiple = false; - } else if (cmp == 0) - multiple = true; - } - - c = &curtiered->iface; - if ((curtiered->current = current) == NULL) { - F_CLR(c, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - return (WT_NOTFOUND); - } - - if (multiple) - F_SET(curtiered, WT_CURTIERED_MULTIPLE); - else - F_CLR(curtiered, WT_CURTIERED_MULTIPLE); - - WT_RET(current->get_key(current, &c->key)); - WT_RET(current->get_value(current, &c->value)); - - F_CLR(c, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if ((*deletedp = __curtiered_deleted(curtiered, &c->value)) == false) - F_SET(c, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); - - return (0); -} - -/* - * __curtiered_compare -- - * WT_CURSOR->compare implementation for the tiered cursor type. - */ -static int -__curtiered_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) -{ - WT_CURSOR_TIERED *atiered; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - atiered = (WT_CURSOR_TIERED *)a; - CURSOR_API_CALL(a, session, compare, NULL); - - /* - * Confirm both cursors refer to the same source and have keys, then compare the keys. - */ - if (strcmp(a->uri, b->uri) != 0) - WT_ERR_MSG(session, EINVAL, "comparison method cursors must reference the same object"); - - WT_ERR(__cursor_needkey(a)); - WT_ERR(__cursor_needkey(b)); - - WT_ERR(__wt_compare(session, atiered->tiered->collator, &a->key, &b->key, cmpp)); - -err: - API_END_RET(session, ret); -} - -/* - * __curtiered_position_tier -- - * Position a tier cursor. - */ -static int -__curtiered_position_tier(WT_CURSOR_TIERED *curtiered, WT_CURSOR *c, bool forward, int *cmpp) -{ - WT_CURSOR *cursor; - WT_SESSION_IMPL *session; - - cursor = &curtiered->iface; - session = CUR2S(cursor); - - c->set_key(c, &cursor->key); - WT_RET(c->search_near(c, cmpp)); - - while (forward ? *cmpp < 0 : *cmpp > 0) { - WT_RET(forward ? c->next(c) : c->prev(c)); - - /* - * With higher isolation levels, where we have stable reads, we're done: the cursor is now - * positioned as expected. - * - * With read-uncommitted isolation, a new record could have appeared in between the search - * and stepping forward / back. In that case, keep going until we see a key in the expected - * range. - */ - if (session->txn->isolation != WT_ISO_READ_UNCOMMITTED) - return (0); - - WT_RET(WT_TIERED_CURCMP(session, curtiered->tiered, c, cursor, *cmpp)); - } - - return (0); -} - -/* - * __curtiered_next -- - * WT_CURSOR->next method for the tiered cursor type. - */ -static int -__curtiered_next(WT_CURSOR *cursor) -{ - WT_CURSOR *c; - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_SESSION_IMPL *session; - u_int i; - int cmp; - bool deleted; - - curtiered = (WT_CURSOR_TIERED *)cursor; - - CURSOR_API_CALL(cursor, session, next, NULL); - __cursor_novalue(cursor); - WT_ERR(__curtiered_enter(curtiered, false)); - - /* If we aren't positioned for a forward scan, get started. */ - if (curtiered->current == NULL || !F_ISSET(curtiered, WT_CURTIERED_ITERATE_NEXT)) { - WT_FORALL_CURSORS(curtiered, c, i) - { - if (!F_ISSET(cursor, WT_CURSTD_KEY_SET)) { - WT_ERR(c->reset(c)); - ret = c->next(c); - } else if (c != curtiered->current && - (ret = __curtiered_position_tier(curtiered, c, true, &cmp)) == 0 && cmp == 0 && - curtiered->current == NULL) - curtiered->current = c; - WT_ERR_NOTFOUND_OK(ret, false); - } - F_SET(curtiered, WT_CURTIERED_ITERATE_NEXT | WT_CURTIERED_MULTIPLE); - F_CLR(curtiered, WT_CURTIERED_ITERATE_PREV); - - /* We just positioned *at* the key, now move. */ - if (curtiered->current != NULL) - goto retry; - } else { -retry: - /* - * If there are multiple cursors on that key, move them forward. - */ - if (F_ISSET(curtiered, WT_CURTIERED_MULTIPLE)) { - WT_FORALL_CURSORS(curtiered, c, i) - { - if (!F_ISSET(c, WT_CURSTD_KEY_INT)) - continue; - if (c != curtiered->current) { - WT_ERR( - WT_TIERED_CURCMP(session, curtiered->tiered, c, curtiered->current, cmp)); - if (cmp == 0) - WT_ERR_NOTFOUND_OK(c->next(c), false); - } - } - } - - /* Move the smallest cursor forward. */ - c = curtiered->current; - WT_ERR_NOTFOUND_OK(c->next(c), false); - } - - /* Find the cursor(s) with the smallest key. */ - if ((ret = __curtiered_get_current(session, curtiered, true, &deleted)) == 0 && deleted) - goto retry; - -err: - __curtiered_leave(curtiered); - if (ret == 0) - __curtiered_deleted_decode(curtiered, &cursor->value); - API_END_RET(session, ret); -} - -/* - * __curtiered_prev -- - * WT_CURSOR->prev method for the tiered cursor type. - */ -static int -__curtiered_prev(WT_CURSOR *cursor) -{ - WT_CURSOR *c; - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_SESSION_IMPL *session; - u_int i; - int cmp; - bool deleted; - - curtiered = (WT_CURSOR_TIERED *)cursor; - - CURSOR_API_CALL(cursor, session, prev, NULL); - __cursor_novalue(cursor); - WT_ERR(__curtiered_enter(curtiered, false)); - - /* If we aren't positioned for a reverse scan, get started. */ - if (curtiered->current == NULL || !F_ISSET(curtiered, WT_CURTIERED_ITERATE_PREV)) { - WT_FORALL_CURSORS(curtiered, c, i) - { - if (!F_ISSET(cursor, WT_CURSTD_KEY_SET)) { - WT_ERR(c->reset(c)); - ret = c->prev(c); - } else if (c != curtiered->current && - (ret = __curtiered_position_tier(curtiered, c, false, &cmp)) == 0 && cmp == 0 && - curtiered->current == NULL) - curtiered->current = c; - WT_ERR_NOTFOUND_OK(ret, false); - } - F_SET(curtiered, WT_CURTIERED_ITERATE_PREV | WT_CURTIERED_MULTIPLE); - F_CLR(curtiered, WT_CURTIERED_ITERATE_NEXT); - - /* We just positioned *at* the key, now move. */ - if (curtiered->current != NULL) - goto retry; - } else { -retry: - /* - * If there are multiple cursors on that key, move them backwards. - */ - if (F_ISSET(curtiered, WT_CURTIERED_MULTIPLE)) { - WT_FORALL_CURSORS(curtiered, c, i) - { - if (!F_ISSET(c, WT_CURSTD_KEY_INT)) - continue; - if (c != curtiered->current) { - WT_ERR( - WT_TIERED_CURCMP(session, curtiered->tiered, c, curtiered->current, cmp)); - if (cmp == 0) - WT_ERR_NOTFOUND_OK(c->prev(c), false); - } - } - } - - /* Move the largest cursor backwards. */ - c = curtiered->current; - WT_ERR_NOTFOUND_OK(c->prev(c), false); - } - - /* Find the cursor(s) with the largest key. */ - if ((ret = __curtiered_get_current(session, curtiered, false, &deleted)) == 0 && deleted) - goto retry; - -err: - __curtiered_leave(curtiered); - if (ret == 0) - __curtiered_deleted_decode(curtiered, &cursor->value); - API_END_RET(session, ret); -} - -/* - * __curtiered_reset -- - * WT_CURSOR->reset method for the tiered cursor type. - */ -static int -__curtiered_reset(WT_CURSOR *cursor) -{ - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - /* - * Don't use the normal __curtiered_enter path: that is wasted work when all we want to do is - * give up our position. - */ - curtiered = (WT_CURSOR_TIERED *)cursor; - CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, reset, NULL); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - - WT_TRET(__curtiered_reset_cursors(curtiered, NULL)); - - /* In case we were left positioned, clear that. */ - __curtiered_leave(curtiered); - -err: - API_END_RET(session, ret); -} - -/* - * __curtiered_lookup -- - * Position a tiered cursor. - */ -static int -__curtiered_lookup(WT_CURSOR_TIERED *curtiered, WT_ITEM *value) -{ - WT_CURSOR *c, *cursor; - WT_DECL_RET; - u_int i; - - c = NULL; - cursor = &curtiered->iface; - - WT_FORALL_CURSORS(curtiered, c, i) - { - c->set_key(c, &cursor->key); - if ((ret = c->search(c)) == 0) { - WT_ERR(c->get_key(c, &cursor->key)); - WT_ERR(c->get_value(c, value)); - if (__curtiered_deleted(curtiered, value)) - ret = WT_NOTFOUND; - goto done; - } - WT_ERR_NOTFOUND_OK(ret, false); - F_CLR(c, WT_CURSTD_KEY_SET); - } - WT_ERR(WT_NOTFOUND); - -done: -err: - if (ret == 0) { - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - F_SET(cursor, WT_CURSTD_KEY_INT); - curtiered->current = c; - if (value == &cursor->value) - F_SET(cursor, WT_CURSTD_VALUE_INT); - } else if (c != NULL) - WT_TRET(c->reset(c)); - - return (ret); -} - -/* - * __curtiered_search -- - * WT_CURSOR->search method for the tiered cursor type. - */ -static int -__curtiered_search(WT_CURSOR *cursor) -{ - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_SESSION_IMPL *session; - - curtiered = (WT_CURSOR_TIERED *)cursor; - - CURSOR_API_CALL(cursor, session, search, NULL); - WT_ERR(__cursor_needkey(cursor)); - __cursor_novalue(cursor); - WT_ERR(__curtiered_enter(curtiered, true)); - F_CLR(curtiered, WT_CURTIERED_ITERATE_NEXT | WT_CURTIERED_ITERATE_PREV); - - ret = __curtiered_lookup(curtiered, &cursor->value); - -err: - __curtiered_leave(curtiered); - if (ret == 0) - __curtiered_deleted_decode(curtiered, &cursor->value); - API_END_RET(session, ret); -} - -/* - * __curtiered_search_near -- - * WT_CURSOR->search_near method for the tiered cursor type. - */ -static int -__curtiered_search_near(WT_CURSOR *cursor, int *exactp) -{ - WT_CURSOR *c, *closest; - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_SESSION_IMPL *session; - u_int i; - int cmp, exact; - bool deleted; - - closest = NULL; - curtiered = (WT_CURSOR_TIERED *)cursor; - exact = 0; - - CURSOR_API_CALL(cursor, session, search_near, NULL); - WT_ERR(__cursor_needkey(cursor)); - __cursor_novalue(cursor); - WT_ERR(__curtiered_enter(curtiered, true)); - F_CLR(curtiered, WT_CURTIERED_ITERATE_NEXT | WT_CURTIERED_ITERATE_PREV); - - /* - * search_near is somewhat fiddly: we can't just use a nearby key from the first tier because - * there could be a closer key in a lower tier. - * - * As we search down the tiers, we stop as soon as we find an exact match. Otherwise, we - * maintain the smallest cursor larger than the search key and the largest cursor smaller than - * the search key. At the end, we prefer the larger cursor, but if no record is larger, position - * on the last record in the tree. - */ - WT_FORALL_CURSORS(curtiered, c, i) - { - c->set_key(c, &cursor->key); - if ((ret = c->search_near(c, &cmp)) == WT_NOTFOUND) { - ret = 0; - continue; - } - if (ret != 0) - goto err; - - /* Do we have an exact match? */ - if (cmp == 0) { - closest = c; - exact = 1; - break; - } - - /* - * Prefer larger cursors. There are two reasons: (1) we expect prefix searches to be a - * common case (as in our own indices); and (2) we need a way to unambiguously know we have - * the "closest" result. - */ - if (cmp < 0) { - if ((ret = c->next(c)) == WT_NOTFOUND) { - ret = 0; - continue; - } - if (ret != 0) - goto err; - } - - /* - * We are trying to find the smallest cursor greater than the search key. - */ - if (closest == NULL) - closest = c; - else { - WT_ERR(WT_TIERED_CURCMP(session, curtiered->tiered, c, closest, cmp)); - if (cmp < 0) - closest = c; - } - } - - /* - * At this point, we either have an exact match, or closest is the smallest cursor larger than - * the search key, or it is NULL if the search key is larger than any record in the tree. - */ - cmp = exact ? 0 : 1; - - /* - * If we land on a deleted item, try going forwards or backwards to find one that isn't deleted. - * If the whole tree is empty, we'll end up with WT_NOTFOUND, as expected. - */ - if (closest == NULL) - deleted = true; - else { - WT_ERR(closest->get_key(closest, &cursor->key)); - WT_ERR(closest->get_value(closest, &cursor->value)); - curtiered->current = closest; - closest = NULL; - deleted = __curtiered_deleted(curtiered, &cursor->value); - if (!deleted) - __curtiered_deleted_decode(curtiered, &cursor->value); - else { - /* - * We have a key pointing at memory that is pinned by the current tier cursor. In the - * unlikely event that we have to reopen cursors to move to the next record, make sure - * the cursor flags are set so a copy is made before the current tier cursor releases - * its position. - */ - F_CLR(cursor, WT_CURSTD_KEY_SET); - F_SET(cursor, WT_CURSTD_KEY_INT); - /* - * We call __curtiered_next here as we want to advance forward. If we are a random - * tiered cursor calling next on the cursor will not advance as we intend. - */ - if ((ret = __curtiered_next(cursor)) == 0) { - cmp = 1; - deleted = false; - } - } - WT_ERR_NOTFOUND_OK(ret, false); - } - if (deleted) { - curtiered->current = NULL; - /* - * We call prev directly here as cursor->prev may be "invalid" if this is a random cursor. - */ - WT_ERR(__curtiered_prev(cursor)); - cmp = -1; - } - *exactp = cmp; - -err: - __curtiered_leave(curtiered); - if (closest != NULL) - WT_TRET(closest->reset(closest)); - - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if (ret == 0) { - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); - } else - curtiered->current = NULL; - - API_END_RET(session, ret); -} - -/* - * __curtiered_put -- - * Put an entry into the primary tree. - */ -static inline int -__curtiered_put(WT_CURSOR_TIERED *curtiered, const WT_ITEM *key, const WT_ITEM *value, - bool position, bool reserve) -{ - WT_CURSOR *primary; - int (*func)(WT_CURSOR *); - - /* - * Clear the existing cursor position. Don't clear the primary cursor: we're about to use it - * anyway. - */ - primary = curtiered->cursors[WT_TIERED_INDEX_LOCAL]; - WT_RET(__curtiered_reset_cursors(curtiered, primary)); - - /* If necessary, set the position for future scans. */ - if (position) - curtiered->current = primary; - - primary->set_key(primary, key); - - /* Our API always leaves the cursor positioned after a reserve call. */ - WT_ASSERT(CUR2S(curtiered), !reserve || position); - func = primary->insert; - if (position) - func = reserve ? primary->reserve : primary->update; - if (!reserve) - primary->set_value(primary, value); - return (func(primary)); -} - -/* - * __curtiered_insert -- - * WT_CURSOR->insert method for the tiered cursor type. - */ -static int -__curtiered_insert(WT_CURSOR *cursor) -{ - WT_CURSOR_TIERED *curtiered; - WT_DECL_ITEM(buf); - WT_DECL_RET; - WT_ITEM value; - WT_SESSION_IMPL *session; - - curtiered = (WT_CURSOR_TIERED *)cursor; - - CURSOR_UPDATE_API_CALL(cursor, session, insert); - WT_ERR(__cursor_needkey(cursor)); - WT_ERR(__cursor_needvalue(cursor)); - WT_ERR(__curtiered_enter(curtiered, false)); - - /* - * It isn't necessary to copy the key out after the lookup in this case because any non-failed - * lookup results in an error, and a failed lookup leaves the original key intact. - */ - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && - (ret = __curtiered_lookup(curtiered, &value)) != WT_NOTFOUND) { - if (ret == 0) - ret = WT_DUPLICATE_KEY; - goto err; - } - - WT_ERR(__curtiered_deleted_encode(session, &cursor->value, &value, &buf)); - WT_ERR(__curtiered_put(curtiered, &cursor->key, &value, false, false)); - - /* - * WT_CURSOR.insert doesn't leave the cursor positioned, and the application may want to free - * the memory used to configure the insert; don't read that memory again (matching the - * underlying file object cursor insert semantics). - */ - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - -err: - __wt_scr_free(session, &buf); - __curtiered_leave(curtiered); - CURSOR_UPDATE_API_END(session, ret); - return (ret); -} - -/* - * __curtiered_update -- - * WT_CURSOR->update method for the tiered cursor type. - */ -static int -__curtiered_update(WT_CURSOR *cursor) -{ - WT_CURSOR_TIERED *curtiered; - WT_DECL_ITEM(buf); - WT_DECL_RET; - WT_ITEM value; - WT_SESSION_IMPL *session; - - curtiered = (WT_CURSOR_TIERED *)cursor; - - CURSOR_UPDATE_API_CALL(cursor, session, update); - WT_ERR(__cursor_needkey(cursor)); - WT_ERR(__cursor_needvalue(cursor)); - WT_ERR(__curtiered_enter(curtiered, false)); - - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { - WT_ERR(__curtiered_lookup(curtiered, &value)); - /* - * Copy the key out, since the insert resets non-primary tier cursors which our lookup may - * have landed on. - */ - WT_ERR(__cursor_needkey(cursor)); - } - WT_ERR(__curtiered_deleted_encode(session, &cursor->value, &value, &buf)); - WT_ERR(__curtiered_put(curtiered, &cursor->key, &value, true, false)); - - /* - * Set the cursor to reference the internal key/value of the positioned cursor. - */ - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - WT_ITEM_SET(cursor->key, curtiered->current->key); - WT_ITEM_SET(cursor->value, curtiered->current->value); - WT_ASSERT(session, F_MASK(curtiered->current, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); - WT_ASSERT(session, F_MASK(curtiered->current, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); - -err: - __wt_scr_free(session, &buf); - __curtiered_leave(curtiered); - CURSOR_UPDATE_API_END(session, ret); - return (ret); -} - -/* - * __curtiered_remove -- - * WT_CURSOR->remove method for the tiered cursor type. - */ -static int -__curtiered_remove(WT_CURSOR *cursor) -{ - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_ITEM value; - WT_SESSION_IMPL *session; - bool positioned; - - curtiered = (WT_CURSOR_TIERED *)cursor; - - /* Check if the cursor is positioned. */ - positioned = F_ISSET(cursor, WT_CURSTD_KEY_INT); - - CURSOR_REMOVE_API_CALL(cursor, session, NULL); - WT_ERR(__cursor_needkey(cursor)); - __cursor_novalue(cursor); - WT_ERR(__curtiered_enter(curtiered, false)); - - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { - WT_ERR(__curtiered_lookup(curtiered, &value)); - /* - * Copy the key out, since the insert resets non-primary tier cursors which our lookup may - * have landed on. - */ - WT_ERR(__cursor_needkey(cursor)); - } - WT_ERR(__curtiered_put(curtiered, &cursor->key, &__tombstone, positioned, false)); - - /* - * If the cursor was positioned, it stays positioned with a key but no value, otherwise, there's - * no position, key or value. This isn't just cosmetic, without a reset, iteration on this - * cursor won't start at the beginning/end of the table. - */ - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if (positioned) - F_SET(cursor, WT_CURSTD_KEY_INT); - else - WT_TRET(cursor->reset(cursor)); - -err: - __curtiered_leave(curtiered); - CURSOR_UPDATE_API_END(session, ret); - return (ret); -} - -/* - * __curtiered_reserve -- - * WT_CURSOR->reserve method for the tiered cursor type. - */ -static int -__curtiered_reserve(WT_CURSOR *cursor) -{ - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_ITEM value; - WT_SESSION_IMPL *session; - - curtiered = (WT_CURSOR_TIERED *)cursor; - - CURSOR_UPDATE_API_CALL(cursor, session, reserve); - WT_ERR(__cursor_needkey(cursor)); - __cursor_novalue(cursor); - WT_ERR(__wt_txn_context_check(session, true)); - WT_ERR(__curtiered_enter(curtiered, false)); - - WT_ERR(__curtiered_lookup(curtiered, &value)); - /* - * Copy the key out, since the insert resets non-primary tier cursors which our lookup may have - * landed on. - */ - WT_ERR(__cursor_needkey(cursor)); - ret = __curtiered_put(curtiered, &cursor->key, NULL, true, true); - -err: - __curtiered_leave(curtiered); - CURSOR_UPDATE_API_END(session, ret); - - /* - * The application might do a WT_CURSOR.get_value call when we return, so we need a value and - * the underlying functions didn't set one up. For various reasons, those functions may not have - * done a search and any previous value in the cursor might race with WT_CURSOR.reserve (and in - * cases like tiered, the reserve never encountered the original key). For simplicity, repeat - * the search here. - */ - return (ret == 0 ? cursor->search(cursor) : ret); -} - -/* - * __curtiered_next_random -- - * WT_CURSOR->next method for the tiered cursor type when configured with next_random. - */ -static int -__curtiered_next_random(WT_CURSOR *cursor) -{ - WT_CURSOR *c; - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_SESSION_IMPL *session; - u_int i, tier; - int exact; - - c = NULL; - curtiered = (WT_CURSOR_TIERED *)cursor; - - CURSOR_API_CALL(cursor, session, next, NULL); - __cursor_novalue(cursor); - WT_ERR(__curtiered_enter(curtiered, false)); - - /* - * Select a random tier. If it is empty, try the next tier and so on, wrapping around until we - * find something or run out of tiers. - */ - tier = __wt_random(&session->rnd) % WT_TIERED_MAX_TIERS; - for (i = 0; i < WT_TIERED_MAX_TIERS; i++) { - c = curtiered->cursors[tier]; - WT_ERR_NOTFOUND_OK(__wt_curfile_next_random(c), true); - if (ret == WT_NOTFOUND) { - if (++tier == WT_TIERED_MAX_TIERS) - tier = 0; - continue; - } - - F_SET(cursor, WT_CURSTD_KEY_INT); - WT_ERR(c->get_key(c, &cursor->key)); - /* - * Search near the current key to resolve any tombstones and position to a valid record. If - * we see a WT_NOTFOUND here that is valid, as the tree has no documents visible to us. - */ - WT_ERR(__curtiered_search_near(cursor, &exact)); - break; - } - -err: - if (ret != 0) { - /* We didn't find a valid record. Don't leave cursor positioned */ - F_CLR(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); - } - __curtiered_leave(curtiered); - API_END_RET(session, ret); -} - -/* - * __curtiered_insert_bulk -- - * WT_CURSOR->insert method for tiered bulk cursors. - */ -static int -__curtiered_insert_bulk(WT_CURSOR *cursor) -{ - WT_CURSOR *bulk_cursor; - WT_CURSOR_TIERED *curtiered; - WT_SESSION_IMPL *session; - - curtiered = (WT_CURSOR_TIERED *)cursor; - session = CUR2S(curtiered); - bulk_cursor = curtiered->cursors[WT_TIERED_INDEX_LOCAL]; - - WT_ASSERT(session, bulk_cursor != NULL); - bulk_cursor->set_key(bulk_cursor, &cursor->key); - bulk_cursor->set_value(bulk_cursor, &cursor->value); - WT_RET(bulk_cursor->insert(bulk_cursor)); - - return (0); -} - -/* - * __curtiered_open_bulk -- - * WT_SESSION->open_cursor method for tiered bulk cursors. - */ -static int -__curtiered_open_bulk(WT_CURSOR_TIERED *curtiered, const char *cfg[]) -{ - WT_CURSOR *cursor; - WT_DATA_HANDLE *dhandle; - WT_DECL_RET; - WT_SESSION_IMPL *session; - WT_TIERED *tiered; - - cursor = &curtiered->iface; - session = CUR2S(curtiered); - tiered = curtiered->tiered; - - /* Bulk cursors only support insert and close. */ - __wt_cursor_set_notsup(cursor); - cursor->insert = __curtiered_insert_bulk; - cursor->close = __wt_curtiered_close; - - WT_ASSERT(session, curtiered->cursors == NULL); - WT_ERR(__wt_calloc_def(session, WT_TIERED_MAX_TIERS, &curtiered->cursors)); - - /* Open a bulk cursor on the local tier. */ - dhandle = tiered->tiers[WT_TIERED_INDEX_LOCAL].tier; - WT_ASSERT(session, dhandle != NULL); - WT_ERR(__wt_open_cursor( - session, dhandle->name, cursor, cfg, &curtiered->cursors[WT_TIERED_INDEX_LOCAL])); - - /* Child cursors always use overwrite and raw mode. */ - F_SET(curtiered->cursors[WT_TIERED_INDEX_LOCAL], WT_CURSTD_OVERWRITE | WT_CURSTD_RAW); - - if (0) { -err: - __wt_free(session, curtiered->cursors); - } - return (ret); -} - -/* - * __wt_curtiered_open -- - * WT_SESSION->open_cursor method for tiered cursors. - */ -int -__wt_curtiered_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], - WT_CURSOR **cursorp) -{ - WT_CONFIG_ITEM cval; - WT_CURSOR_STATIC_INIT(iface, __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curtiered_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curtiered_next, /* next */ - __curtiered_prev, /* prev */ - __curtiered_reset, /* reset */ - __curtiered_search, /* search */ - __curtiered_search_near, /* search-near */ - __curtiered_insert, /* insert */ - __wt_cursor_modify_value_format_notsup, /* modify */ - __curtiered_update, /* update */ - __curtiered_remove, /* remove */ - __curtiered_reserve, /* reserve */ - __wt_cursor_reconfigure, /* reconfigure */ - __wt_cursor_notsup, /* cache */ - __wt_cursor_reopen_notsup, /* reopen */ - __wt_curtiered_close); /* close */ - WT_CURSOR *cursor; - WT_CURSOR_TIERED *curtiered; - WT_DECL_RET; - WT_TIERED *tiered; - bool bulk; - - WT_STATIC_ASSERT(offsetof(WT_CURSOR_TIERED, iface) == 0); - - curtiered = NULL; - cursor = NULL; - tiered = NULL; - - if (!WT_PREFIX_MATCH(uri, "tiered:")) - return (__wt_unexpected_object_type(session, uri, "tiered:")); - - WT_RET(__wt_config_gets_def(session, cfg, "checkpoint", 0, &cval)); - if (cval.len != 0) - WT_RET_MSG(session, EINVAL, "tiered tables do not support opening by checkpoint"); - - WT_RET(__wt_config_gets_def(session, cfg, "bulk", 0, &cval)); - bulk = cval.val != 0; - - /* Get the tiered data handle. */ - ret = __wt_session_get_dhandle(session, uri, NULL, cfg, bulk ? WT_DHANDLE_EXCLUSIVE : 0); - - /* Check whether the exclusive open for a bulk load succeeded. */ - if (bulk && ret == EBUSY) - ret = EINVAL; - /* Flag any errors from the tree get. */ - WT_ERR(ret); - - tiered = (WT_TIERED *)session->dhandle; - - /* Make sure we have exclusive access if and only if we want it */ - WT_ASSERT(session, !bulk || tiered->iface.excl_session != NULL); - - WT_ERR(__wt_calloc_one(session, &curtiered)); - cursor = (WT_CURSOR *)curtiered; - *cursor = iface; - cursor->session = (WT_SESSION *)session; - WT_ERR(__wt_strdup(session, tiered->iface.name, &cursor->uri)); - cursor->key_format = tiered->key_format; - cursor->value_format = tiered->value_format; - - curtiered->tiered = tiered; - tiered = NULL; - - /* If the next_random option is set, configure a random cursor */ - WT_ERR(__wt_config_gets_def(session, cfg, "next_random", 0, &cval)); - if (cval.val != 0) { - __wt_cursor_set_notsup(cursor); - cursor->next = __curtiered_next_random; - } - - WT_ERR(__wt_cursor_init(cursor, cursor->uri, owner, cfg, cursorp)); - - if (bulk) - WT_ERR(__curtiered_open_bulk(curtiered, cfg)); - - if (0) { -err: - if (curtiered != NULL) - WT_TRET(__wt_curtiered_close(cursor)); - else if (tiered != NULL) - WT_WITH_DHANDLE( - session, (WT_DATA_HANDLE *)tiered, WT_TRET(__wt_session_release_dhandle(session))); - - *cursorp = NULL; - } - - return (ret); -} diff --git a/src/third_party/wiredtiger/src/tiered/tiered_handle.c b/src/third_party/wiredtiger/src/tiered/tiered_handle.c index 363a9c97140..11ecd0a4941 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_handle.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_handle.c @@ -138,7 +138,7 @@ __tiered_create_object(WT_SESSION_IMPL *session, WT_TIERED *tiered) orig_name = tiered->tiers[WT_TIERED_INDEX_LOCAL].name; /* * If we have an existing local file in the tier, alter the table to indicate this one is now - * readonly. + * readonly. We are already holding the schema lock so we can call alter. */ if (orig_name != NULL) { cfg[0] = "readonly=true"; @@ -161,13 +161,6 @@ __tiered_create_object(WT_SESSION_IMPL *session, WT_TIERED *tiered) /* Create the new shared object. */ WT_ERR(__wt_schema_create(session, name, config)); -#if 0 - /* - * If we get here we have successfully created the object. It is ready to be fully flushed to - * the cloud. Push a work element to let the internal thread do that here. - */ -#endif - err: __wt_free(session, config); __wt_free(session, name); @@ -313,10 +306,6 @@ static int __tiered_switch(WT_SESSION_IMPL *session, const char *config) { WT_DECL_RET; -#if 0 - WT_FILE_SYSTEM *fs; - WT_STORAGE_SOURCE *storage_source; -#endif WT_TIERED *tiered; bool need_object, need_tree, tracking; @@ -357,42 +346,22 @@ __tiered_switch(WT_SESSION_IMPL *session, const char *config) WT_RET(__wt_meta_track_on(session)); tracking = true; - /* Create the object: entry in the metadata. */ - if (need_object) - WT_ERR(__tiered_create_object(session, tiered)); - if (need_tree) WT_ERR(__tiered_create_tier_tree(session, tiered)); + /* Create the object: entry in the metadata. */ + if (need_object) { + WT_ERR(__tiered_create_object(session, tiered)); +#if 1 + WT_ERR(__wt_tiered_put_flush(session, tiered)); +#else + WT_ERR(__wt_tier_flush(session, tiered, tiered->current_id)); +#endif + } + /* We always need to create a local object. */ WT_ERR(__tiered_create_local(session, tiered)); -#if 0 - /* - * We expect this part to be done asynchronously in its own thread. First flush the contents of - * the data file to the new cloud object. - */ - storage_source = tiered->bstorage->storage_source; - fs = tiered->bucket_storage->file_system; - WT_ASSERT(session, storage_source != NULL); - - /* This call make take a while, and may fail due to network timeout. */ - WT_ERR(storage_source->ss_flush(storage_source, &session->iface, - fs, old_filename, object_name, NULL)); - - /* - * The metadata for the old local object will be initialized with "flush=0". When the flush call - * completes, it can be marked as "flush=1". When that's done, we can finish the flush. The - * flush finish call moves the file from the home directory to the extension's cache. Then the - * extension will own it. - * - * We may need a way to restart flushes for those not completed (after a crash), or failed (due - * to previous network outage). - */ - WT_ERR(storage_source->ss_flush_finish(storage_source, &session->iface, - fs, old_filename, object_name, NULL)); -#endif - /* Update the tiered: metadata to new object number and tiered array. */ WT_ERR(__tiered_update_metadata(session, tiered, config)); tracking = false; @@ -485,7 +454,10 @@ __tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) WT_DECL_ITEM(tmp); WT_DECL_RET; WT_TIERED *tiered; +#if 1 + WT_TIERED_WORK_UNIT *entry; uint32_t unused; +#endif char *metaconf; const char *obj_cfg[] = {WT_CONFIG_BASE(session, object_meta), NULL, NULL}; const char **tiered_cfg, *config; @@ -549,10 +521,17 @@ __tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) __wt_free(session, dhandle->cfg[1]); dhandle->cfg[1] = metaconf; } +#if 1 if (0) { /* Temp code to keep s_all happy. */ FLD_SET(unused, WT_TIERED_OBJ_LOCAL | WT_TIERED_TREE_UNUSED); + FLD_SET(unused, WT_TIERED_WORK_FORCE | WT_TIERED_WORK_FREE); + WT_ERR(__wt_tiered_put_drop_local(session, tiered, tiered->current_id)); + WT_ERR(__wt_tiered_put_drop_shared(session, tiered, tiered->current_id)); + __wt_tiered_get_drop_local(session, 0, &entry); + __wt_tiered_get_drop_shared(session, &entry); } +#endif if (0) { err: diff --git a/src/third_party/wiredtiger/src/tiered/tiered_work.c b/src/third_party/wiredtiger/src/tiered/tiered_work.c new file mode 100644 index 00000000000..728a7a0b3b2 --- /dev/null +++ b/src/third_party/wiredtiger/src/tiered/tiered_work.c @@ -0,0 +1,151 @@ +/*- + * Copyright (c) 2014-present MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_tiered_push_work -- + * Push a work unit to the queue. Assumes it is passed an already filled out structure. + */ +void +__wt_tiered_push_work(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT *entry) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + __wt_spin_lock(session, &conn->tiered_lock); + TAILQ_INSERT_TAIL(&conn->tieredqh, entry, q); + WT_STAT_CONN_INCR(session, tiered_work_units_created); + __wt_spin_unlock(session, &conn->tiered_lock); + __wt_cond_signal(session, conn->tiered_cond); + return; +} + +/* + * __wt_tiered_pop_work -- + * Pop a work unit of the given type from the queue. If a maximum value is given, only return a + * work unit that is less than the maximum value. The caller is responsible for freeing the + * returned work unit structure. + */ +void +__wt_tiered_pop_work( + WT_SESSION_IMPL *session, uint32_t type, uint64_t maxval, WT_TIERED_WORK_UNIT **entryp) +{ + WT_CONNECTION_IMPL *conn; + WT_TIERED_WORK_UNIT *entry; + + *entryp = entry = NULL; + + conn = S2C(session); + if (TAILQ_EMPTY(&conn->tieredqh)) + return; + __wt_spin_lock(session, &conn->tiered_lock); + + TAILQ_FOREACH (entry, &conn->tieredqh, q) { + if (FLD_ISSET(type, entry->type) && (maxval == 0 || entry->op_val < maxval)) { + TAILQ_REMOVE(&conn->tieredqh, entry, q); + WT_STAT_CONN_INCR(session, tiered_work_units_dequeued); + break; + } + } + *entryp = entry; + __wt_spin_unlock(session, &conn->tiered_lock); + return; +} + +/* + * __wt_tiered_get_flush -- + * Get the first flush work unit from the queue. The id information cannot change between our + * caller and here. The caller is responsible for freeing the work unit. + */ +void +__wt_tiered_get_flush(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT **entryp) +{ + __wt_tiered_pop_work(session, WT_TIERED_WORK_FLUSH, 0, entryp); + return; +} + +/* + * __wt_tiered_get_drop_local -- + * Get a drop local work unit if it is less than the time given. The caller is responsible for + * freeing the work unit. + */ +void +__wt_tiered_get_drop_local(WT_SESSION_IMPL *session, uint64_t now, WT_TIERED_WORK_UNIT **entryp) +{ + __wt_tiered_pop_work(session, WT_TIERED_WORK_DROP_LOCAL, now, entryp); + return; +} + +/* + * __wt_tiered_get_drop_shared -- + * Get a drop shared work unit. The caller is responsible for freeing the work unit. + */ +void +__wt_tiered_get_drop_shared(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT **entryp) +{ + __wt_tiered_pop_work(session, WT_TIERED_WORK_DROP_SHARED, 0, entryp); + return; +} + +/* + * __wt_tiered_put_drop_local -- + * Add a drop local work unit for the given ID to the queue. + */ +int +__wt_tiered_put_drop_local(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint64_t id) +{ + WT_TIERED_WORK_UNIT *entry; + uint64_t now; + + WT_RET(__wt_calloc_one(session, &entry)); + entry->type = WT_TIERED_WORK_DROP_LOCAL; + entry->id = id; + WT_ASSERT(session, tiered->bstorage != NULL); + __wt_seconds(session, &now); + /* Put a work unit in the queue with the time this object expires. */ + entry->op_val = now + tiered->bstorage->retain_secs; + entry->tiered = tiered; + __wt_tiered_push_work(session, entry); + return (0); +} + +/* + * __wt_tiered_put_drop_shared -- + * Add a drop shared work unit for the given ID to the queue. + */ +int +__wt_tiered_put_drop_shared(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint64_t id) +{ + WT_TIERED_WORK_UNIT *entry; + + WT_RET(__wt_calloc_one(session, &entry)); + entry->type = WT_TIERED_WORK_DROP_SHARED; + entry->id = id; + entry->tiered = tiered; + __wt_tiered_push_work(session, entry); + return (0); +} + +/* + * __wt_tiered_put_flush -- + * Add a flush work unit to the queue. We're single threaded so the tiered structure's id + * information cannot change between our caller and here. + */ +int +__wt_tiered_put_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered) +{ + WT_TIERED_WORK_UNIT *entry; + + WT_RET(__wt_calloc_one(session, &entry)); + entry->type = WT_TIERED_WORK_FLUSH; + entry->id = tiered->current_id; + entry->tiered = tiered; + __wt_tiered_push_work(session, entry); + return (0); +} diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 61720a8adaa..a8c5c8664a4 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -64,14 +64,15 @@ __checkpoint_name_check(WT_SESSION_IMPL *session, const char *uri) WT_ERR(cursor->get_key(cursor, &uri)); if (!WT_PREFIX_MATCH(uri, "colgroup:") && !WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "index:") && !WT_PREFIX_MATCH(uri, WT_SYSTEM_PREFIX) && - !WT_PREFIX_MATCH(uri, "table:")) { + !WT_PREFIX_MATCH(uri, "table:") && !WT_PREFIX_MATCH(uri, "tiered:")) { fail = uri; break; } } WT_ERR_NOTFOUND_OK(ret, false); } else if (!WT_PREFIX_MATCH(uri, "colgroup:") && !WT_PREFIX_MATCH(uri, "file:") && - !WT_PREFIX_MATCH(uri, "index:") && !WT_PREFIX_MATCH(uri, "table:")) + !WT_PREFIX_MATCH(uri, "index:") && !WT_PREFIX_MATCH(uri, "table:") && + !WT_PREFIX_MATCH(uri, "tiered:")) fail = uri; if (fail != NULL) @@ -254,8 +255,7 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) } /* Should not be called with anything other than a live btree handle. */ - WT_ASSERT(session, - session->dhandle->type == WT_DHANDLE_TYPE_BTREE && session->dhandle->checkpoint == NULL); + WT_ASSERT(session, WT_DHANDLE_BTREE(session->dhandle) && session->dhandle->checkpoint == NULL); btree = S2BT(session); diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 5b5482524f9..ee304fedff5 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -1375,6 +1375,31 @@ err: } /* + * __rollback_progress_msg -- + * Log a verbose message about the progress of the current rollback to stable. + */ +static void +__rollback_progress_msg(WT_SESSION_IMPL *session, struct timespec rollback_start, + uint64_t rollback_count, uint64_t *rollback_msg_count) +{ + struct timespec cur_time; + uint64_t time_diff; + + __wt_epoch(session, &cur_time); + + /* Time since the rollback started. */ + time_diff = WT_TIMEDIFF_SEC(cur_time, rollback_start); + + if ((time_diff / WT_PROGRESS_MSG_PERIOD) > *rollback_msg_count) { + __wt_verbose(session, WT_VERB_RECOVERY_PROGRESS, + "Rollback to stable has been running for %" PRIu64 " seconds and has inspected %" PRIu64 + " files. For more detailed logging, enable WT_VERB_RTS", + time_diff, rollback_count); + ++(*rollback_msg_count); + } +} + +/* * __rollback_to_stable_btree_apply -- * Perform rollback to stable to all files listed in the metadata, apart from the metadata and * history store files. @@ -1389,7 +1414,8 @@ __rollback_to_stable_btree_apply(WT_SESSION_IMPL *session) WT_TXN_GLOBAL *txn_global; wt_timestamp_t max_durable_ts, newest_start_durable_ts, newest_stop_durable_ts, rollback_timestamp; - uint64_t rollback_txnid; + struct timespec rollback_timer; + uint64_t rollback_count, rollback_msg_count, rollback_txnid; uint32_t btree_id; size_t addr_size; char ts_string[2][WT_TS_INT_STRING_SIZE]; @@ -1398,9 +1424,12 @@ __rollback_to_stable_btree_apply(WT_SESSION_IMPL *session) bool dhandle_allocated, perform_rts; txn_global = &S2C(session)->txn_global; - rollback_txnid = 0; + rollback_count = rollback_msg_count = rollback_txnid = 0; addr_size = 0; + /* Initialize the verbose tracking timer. */ + __wt_epoch(session, &rollback_timer); + /* * Copy the stable timestamp, otherwise we'd need to lock it each time it's accessed. Even * though the stable timestamp isn't supposed to be updated while rolling back, accessing it @@ -1423,6 +1452,10 @@ __rollback_to_stable_btree_apply(WT_SESSION_IMPL *session) S2C(session)->recovery_ckpt_snapshot_count); while ((ret = cursor->next(cursor)) == 0) { + /* Log a progress message. */ + __rollback_progress_msg(session, rollback_timer, rollback_count, &rollback_msg_count); + ++rollback_count; + WT_ERR(cursor->get_key(cursor, &uri)); dhandle_allocated = perform_rts = false; diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index eec4fe99cc2..e0953c0abfe 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -101,8 +101,8 @@ list_init_block(WT_SESSION *session, const char *key, WT_BLOCK *block) else if (ret != WT_NOTFOUND) WT_ERR(util_err(session, ret, "WT_CONFIG_PARSER.get")); - if ((ret = parser->get(parser, "block_allocation", &cval)) == 0) - block->log_structured = WT_STRING_MATCH("log_structured", cval.str, cval.len); + if (WT_PREFIX_MATCH(key, "tiered:")) + block->has_objects = true; err: if (parser != NULL && (tret = parser->close(parser)) != 0) { diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/smoke.sh b/src/third_party/wiredtiger/test/csuite/incr_backup/smoke.sh index 65727df015e..7816a4ec70b 100755 --- a/src/third_party/wiredtiger/test/csuite/incr_backup/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/incr_backup/smoke.sh @@ -4,9 +4,15 @@ set -e # Smoke-test incr-backup as part of running "make check". -# If $top_builddir/$top_srcdir aren't set, default to building in build_posix -# and running in test/csuite. -top_builddir=${top_builddir:-../../build_posix} -top_srcdir=${top_srcdir:-../..} +if [ -n "$1" ] +then + # If the test binary is passed in manually. + $TEST_WRAPPER $1 -v 3 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} -$TEST_WRAPPER $top_builddir/test/csuite/test_incr_backup -v 3 + $TEST_WRAPPER $top_builddir/test/csuite/test_incr_backup -v 3 +fi diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh index 713b000b4f1..bfa8656f6b1 100755 --- a/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/random_abort/smoke.sh @@ -4,12 +4,18 @@ set -e # Smoke-test random-abort as part of running "make check". -# If $top_builddir/$top_srcdir aren't set, default to building in build_posix -# and running in test/csuite. -top_builddir=${top_builddir:-../../build_posix} -top_srcdir=${top_srcdir:-../..} - -$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -t 10 -T 5 -$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -m -t 10 -T 5 -$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -C -t 10 -T 5 -$TEST_WRAPPER $top_builddir/test/csuite/test_random_abort -C -m -t 10 -T 5 +if [ -n "$1" ] +then + # If the test binary is passed in manually. + test_bin=$1 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_random_abort +fi +$TEST_WRAPPER $test_bin -t 10 -T 5 +$TEST_WRAPPER $test_bin -m -t 10 -T 5 +$TEST_WRAPPER $test_bin -C -t 10 -T 5 +$TEST_WRAPPER $test_bin -C -m -t 10 -T 5 diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh index 7d5b96d9b57..9c7c043beb0 100755 --- a/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/random_directio/smoke.sh @@ -9,8 +9,12 @@ set -e top_builddir=${top_builddir:-../../build_posix} top_srcdir=${top_srcdir:-../..} -RUN_TEST_CMD="$TEST_WRAPPER $top_builddir/test/csuite/test_random_directio" - +if [ -n "$1" ] +then + RUN_TEST_CMD="$TEST_WRAPPER $1" +else + RUN_TEST_CMD="$TEST_WRAPPER $top_builddir/test/csuite/test_random_directio" +fi # Replace for more complete testing #TEST_THREADS="1 5 10" TEST_THREADS="5" diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh index 78effe1acab..5e82ae180bc 100755 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh @@ -4,15 +4,23 @@ set -e # Smoke-test schema-abort as part of running "make check". -# If $top_builddir/$top_srcdir aren't set, default to building in build_posix -# and running in test/csuite. -top_builddir=${top_builddir:-../../build_posix} -top_srcdir=${top_srcdir:-../..} -$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -t 10 -T 5 -$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -C -t 10 -T 5 -$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -C -m -t 10 -T 5 +if [ -n "$1" ] +then + # If the test binary is passed in manually. + test_bin=$1 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_schema_abort +fi + +$TEST_WRAPPER $test_bin -t 10 -T 5 +$TEST_WRAPPER $test_bin -m -t 10 -T 5 +$TEST_WRAPPER $test_bin -C -t 10 -T 5 +$TEST_WRAPPER $test_bin -C -m -t 10 -T 5 # FIXME: In WT-6116 the test is failing if timestamps are turned off. -#$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -z -$TEST_WRAPPER $top_builddir/test/csuite/test_schema_abort -m -t 10 -T 5 -x +#$TEST_WRAPPER $test_bin -m -t 10 -T 5 -z +$TEST_WRAPPER $test_bin -m -t 10 -T 5 -x diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh index 0cac6a56ea4..18d7f9b8dae 100755 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh @@ -6,21 +6,25 @@ set -e # to add a stress timing in checkpoint prepare. default_test_args="-t 10 -T 5" -while getopts ":s" opt; do +while getopts ":sb:" opt; do case $opt in s) default_test_args="$default_test_args -s" ;; + b) test_bin=$OPTARG ;; esac done +if [ -z "$test_bin" ] +then + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_timestamp_abort +fi -# If $top_builddir/$top_srcdir aren't set, default to building in build_posix -# and running in test/csuite. -top_builddir=${top_builddir:-../../build_posix} -top_srcdir=${top_srcdir:-../..} - -$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort $default_test_args -#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort $default_test_args -L -$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m $default_test_args -#$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -m $default_test_args -L -$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C $default_test_args -$TEST_WRAPPER $top_builddir/test/csuite/test_timestamp_abort -C -m $default_test_args +$TEST_WRAPPER $test_bin $default_test_args +#$TEST_WRAPPER $test_bin $default_test_args -L +$TEST_WRAPPER $test_bin -m $default_test_args +#$TEST_WRAPPER $test_bin -m $default_test_args -L +$TEST_WRAPPER $test_bin -C $default_test_args +$TEST_WRAPPER $test_bin -C -m $default_test_args diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index c830023c801..e6111dc937b 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -301,6 +301,10 @@ run_check_subtest( subtest_args[narg++] = (char *)"subtest"; subtest_args[narg++] = (char *)"-h"; subtest_args[narg++] = opts->home; + if (opts->build_dir != NULL) { + subtest_args[narg++] = (char *)"-b"; + subtest_args[narg++] = opts->build_dir; + } subtest_args[narg++] = (char *)"-v"; /* subtest is always verbose */ subtest_args[narg++] = (char *)"-p"; subtest_args[narg++] = (char *)"-o"; @@ -498,9 +502,10 @@ subtest_main(int argc, char *argv[], bool close_test) testutil_check(__wt_snprintf(filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE)); testutil_assert(freopen(filename, "a", stdout) != NULL); +#ifndef WT_FAIL_FS_LIB #define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" - - testutil_build_dir(buf, 1024); +#endif + testutil_build_dir(opts, buf, 1024); testutil_check(__wt_snprintf(config, sizeof(config), "create,cache_size=250M,log=(enabled),transaction_sync=(enabled,method=none),extensions=(%s/" "%s=(early_load,config={environment=true,verbose=true}))", diff --git a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c index ac7dfc4e6b9..140d24d55d1 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c @@ -46,8 +46,10 @@ main(int argc, char *argv[]) testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); +#ifndef WT_FAIL_FS_LIB #define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" - testutil_build_dir(buf, 1024); +#endif + testutil_build_dir(opts, buf, 1024); testutil_check(__wt_snprintf( config, sizeof(config), "create,extensions=(%s/%s=(early_load=true))", buf, WT_FAIL_FS_LIB)); testutil_check(wiredtiger_open(opts->home, NULL, config, &opts->conn)); diff --git a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c index ed4b8d63869..4eec8ad369e 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c @@ -31,6 +31,7 @@ #define MAXKEY 10000 #define PERIOD 60 +#define HOME_LEN 256 static WT_CONNECTION *conn; static uint64_t worker, worker_busy, verify, verify_busy; @@ -38,6 +39,8 @@ static u_int workers, uris; static bool done = false; static bool verbose = false; static char *uri_list[750]; +static char home[HOME_LEN]; +extern char *__wt_optarg; static void uri_init(void) @@ -248,11 +251,10 @@ runone(bool config_cache) { pthread_t idlist[1000]; u_int i, j; - char buf[256], home[256]; + char buf[256]; done = false; - testutil_work_dir_from_path(home, sizeof(home), "WT_TEST.wt4333_handle_locks"); testutil_make_work_dir(home); testutil_check(__wt_snprintf(buf, sizeof(buf), @@ -320,23 +322,32 @@ run(int argc, char *argv[]) WT_RAND_STATE rnd; u_int i, n; int ch; + bool default_home; (void)testutil_set_progname(argv); __wt_random_init_seed(NULL, &rnd); - while ((ch = __wt_getopt(argv[0], argc, argv, "v")) != EOF) { + default_home = true; + while ((ch = __wt_getopt(argv[0], argc, argv, "vh:")) != EOF) { switch (ch) { case 'v': verbose = true; break; + case 'h': + strncpy(home, __wt_optarg, HOME_LEN); + home[HOME_LEN - 1] = '\0'; + default_home = false; + break; default: fprintf(stderr, "usage: %s [-v]\n", argv[0]); return (EXIT_FAILURE); } } - (void)signal(SIGALRM, on_alarm); + if (default_home) + testutil_work_dir_from_path(home, sizeof(home), "WT_TEST.wt4333_handle_locks"); + /* Each test in the table runs for a minute, run 5 tests at random. */ for (i = 0; i < 5; ++i) { n = __wt_random(&rnd) % WT_ELEMENTS(runs); diff --git a/src/third_party/wiredtiger/test/ctest_helpers.cmake b/src/third_party/wiredtiger/test/ctest_helpers.cmake new file mode 100644 index 00000000000..08584b719ca --- /dev/null +++ b/src/third_party/wiredtiger/test/ctest_helpers.cmake @@ -0,0 +1,135 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# create_test_executable(target SOURCES <source files> [EXECUTABLE_NAME <name>] [BINARY_DIR <dir>] [INCLUDES <includes>] +# [ADDITIONAL_FILES <files>] [ADDITIONAL_DIRECTORIES <dirs>] [LIBS <libs>] [FLAGS <flags>]) +# Defines a C test executable binary. This helper does the necessary initialisation to ensure the correct flags and libraries +# are used when compiling the test executable. +# target - Target name of the test. +# SOURCES <source files> - Sources to compile for the given test. +# EXECUTABLE_NAME <name> - A name for the output test binary. Defaults to the target name if not given. +# BINARY_DIR <dir> - The output directory to install the binaries. Defaults to 'CMAKE_CURRENT_BINARY_DIR' if not given. +# INCLUDES <includes> - Additional includes for building the test binary. +# ADDITIONAL_FILES <files> - Additional files, scripts, etc we want to copy over to the output test binary. Useful if we need +# to setup an additional wrappers needed to run the test. +# ADDITIONAL_DIRECTORIES <dirs> - Additional directories we want to copy over to the output test binary. Useful if we need +# to setup an additional configs and environments needed to run the test. +# LIBS <libs> - Additional libs to link to the test binary. +# FLAGS <flags> - Additional flags to compile the test binary with. +function(create_test_executable target) + cmake_parse_arguments( + PARSE_ARGV + 1 + "CREATE_TEST" + "CXX" + "EXECUTABLE_NAME;BINARY_DIR" + "SOURCES;INCLUDES;ADDITIONAL_FILES;ADDITIONAL_DIRECTORIES;LIBS;FLAGS" + ) + if (NOT "${CREATE_TEST_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to create_test_executable: ${CREATE_TEST_UNPARSED_ARGUMENTS}") + endif() + if ("${CREATE_TEST_SOURCES}" STREQUAL "") + message(FATAL_ERROR "No sources given to create_test_executable") + endif() + + set(test_binary_dir "${CMAKE_CURRENT_BINARY_DIR}") + # Allow the user to specify a custom binary directory. + if(NOT "${CREATE_TEST_BINARY_DIR}" STREQUAL "") + set(test_binary_dir "${CREATE_TEST_BINARY_DIR}") + endif() + + # Define our test executable. + add_executable(${target} ${CREATE_TEST_SOURCES}) + # If we want the output binary to be a different name than the target. + if (NOT "${CREATE_TEST_EXECUTABLE_NAME}" STREQUAL "") + set_target_properties(${target} + PROPERTIES + OUTPUT_NAME "${CREATE_TEST_EXECUTABLE_NAME}" + ) + endif() + set_target_properties(${target} + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${test_binary_dir}" + ) + + # Append the necessary compiler flags. + if(NOT CREATE_TEST_CXX) + # Don't append the strict diagnostic flags to C++ targets (as these are chosen for C targets). + set(test_c_flags "${COMPILER_DIAGNOSTIC_FLAGS}") + endif() + if(NOT "${CREATE_TEST_FLAGS}" STREQUAL "") + list(APPEND test_c_flags ${CREATE_TEST_FLAGS}) + endif() + target_compile_options(${target} PRIVATE ${test_c_flags}) + + # Include the base set of directories for a wiredtiger C test. + target_include_directories(${target} + PRIVATE + ${CMAKE_SOURCE_DIR}/src/include + ${CMAKE_SOURCE_DIR}/test/utility + ${CMAKE_BINARY_DIR}/config + ) + if(NOT "${CREATE_TEST_INCLUDES}" STREQUAL "") + target_include_directories(${target} PRIVATE ${CREATE_TEST_INCLUDES}) + endif() + + # Link the base set of libraries for a wiredtiger C test. + target_link_libraries(${target} wiredtiger test_util) + if(NOT "${CREATE_TEST_LIBS}" STREQUAL "") + target_link_libraries(${target} ${CREATE_TEST_LIBS}) + endif() + + # Install any additional files, scripts, etc in the output test binary + # directory. Useful if we need to setup an additional wrappers needed to run the test + # executable. + foreach(file IN LISTS CREATE_TEST_ADDITIONAL_FILES) + get_filename_component(file_basename ${file} NAME) + # Copy the file to the given test/targets build directory. + add_custom_command(OUTPUT ${test_binary_dir}/${file_basename} + COMMAND ${CMAKE_COMMAND} -E copy + ${file} + ${test_binary_dir}/${file_basename} + DEPENDS ${file} + ) + add_custom_target(copy_file_${target}_${file_basename} DEPENDS ${test_binary_dir}/${file_basename}) + add_dependencies(${target} copy_file_${target}_${file_basename}) + endforeach() + # Install any additional directories in the output test binary directory. + # Useful if we need to setup an additional configs and environments needed to run the test executable. + foreach(dir IN LISTS CREATE_TEST_ADDITIONAL_DIRECTORIES) + get_filename_component(dir_basename ${dir} NAME) + # Copy the file to the given test/targets build directory. + add_custom_command(OUTPUT ${test_binary_dir}/${dir_basename} + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${dir} + ${test_binary_dir}/${dir_basename} + ) + add_custom_target(copy_dir_${target}_${dir_basename} DEPENDS ${test_binary_dir}/${dir_basename}) + add_dependencies(${target} copy_dir_${target}_${dir_basename}) + endforeach() +endfunction() diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 6e8c3f01d11..27233ca996c 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -112,7 +112,7 @@ functions: if [ "$OS" != "Windows_NT" ]; then CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fno-omit-frame-pointer -fsanitize=address" CXXFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb -fPIC" \ ../configure ${configure_python_setting|} \ - --enable-diagnostic --with-builtins=lz4,snappy,zlib + --enable-diagnostic --with-builtins=lz4,snappy,zlib,zstd fi - *make_wiredtiger "compile wiredtiger with builtins": @@ -415,7 +415,7 @@ functions: if [ -d "WT_TEST" ]; then # Dump stderr/stdout contents generated by the C libraries onto console for Python tests - find "WT_TEST" -name "std*.txt" ! -empty -exec sh -c "echo 'Contents from {}:'; cat {}" \; + find "WT_TEST" -name "std*.txt" ! -empty -exec sh -c "echo 'Contents from {}:'; cat '{}'" \; fi "checkpoint test": @@ -519,7 +519,7 @@ variables: configure_env_vars: CC="/opt/mongodbtoolchain/v3/bin/clang -fsanitize=address" PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fno-omit-frame-pointer" CXXFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb -fPIC" - posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib + posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib,zstd - func: "format test script" vars: format_test_script_args: -R -t 360 @@ -1502,6 +1502,16 @@ tasks: - func: "compile wiredtiger" - func: "unit test" + - name: unit-test-zstd + tags: ["python"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "unit test" + vars: + unit_test_args: -v 2 --zstd + - name: unit-test-long tags: ["python"] depends_on: @@ -2041,7 +2051,7 @@ tasks: - func: "compile wiredtiger" vars: configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH ADD_CFLAGS="--coverage -fPIC -ggdb" LDFLAGS=--coverage - posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-python --with-builtins=lz4,snappy,zlib + posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-python --with-builtins=lz4,snappy,zlib,zstd - func: "make check all" - func: "unit test" vars: @@ -2301,7 +2311,7 @@ tasks: - func: "get project" - func: "compile wiredtiger" vars: - posix_configure_flags: --enable-strict --disable-shared --with-builtins=snappy,zlib + posix_configure_flags: --enable-strict --disable-shared --with-builtins=snappy,zlib,zstd - command: shell.exec params: working_dir: "wiredtiger/build_posix" @@ -2545,7 +2555,9 @@ buildvariants: - name: syscall-linux - name: configure-combinations - name: checkpoint-filetypes-test + - name: unit-test-zstd - name: unit-test-long + distros: ubuntu1804-large - name: unit-test-random-seed - name: spinlock-gcc-test - name: spinlock-pthread-adaptive-test @@ -2668,7 +2680,9 @@ buildvariants: distros: rhel80-build - name: syscall-linux - name: checkpoint-filetypes-test + - name: unit-test-zstd - name: unit-test-long + distros: rhel80-large - name: spinlock-gcc-test - name: spinlock-pthread-adaptive-test - name: compile-wtperf diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh index 4c62d379d38..78e5ee2e2e1 100755 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -68,6 +68,7 @@ run_format() flags="-1q $(bflag $1)" args="" + args+="runs.type=row " # Temporarily disable column store tests args+="cache=80 " # Medium cache so there's eviction args+="checkpoints=1 " # Force periodic writes args+="compression=snappy " # We only built with snappy, force the choice @@ -80,6 +81,8 @@ run_format() args+="rows=1000000 " args+="salvage=0 " # Faster runs args+="timer=4 " + args+="transaction.isolation=snapshot " + args+="transaction.timestamps=1 " args+="verify=0 " # Faster runs for am in $2; do diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index f7321f77c99..e6d9b6a9ae0 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -611,6 +611,15 @@ config_directio(void) } /* + * Direct I/O may not work with imports for the same reason as for backups. + */ + if (g.c_import) { + if (config_is_perm("import")) + testutil_die(EINVAL, "direct I/O is incompatible with import configurations"); + config_single("import=0", false); + } + + /* * Direct I/O may not work with mmap. Theoretically, Linux ignores direct I/O configurations in * the presence of shared cache configurations (including mmap), but we've seen file corruption * and it doesn't make much sense (the library disallows the combination). diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 8d74b84259d..d173fb7c10b 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -52,8 +52,11 @@ wt_3rdpartydir = os.path.join(wt_disttop, 'test', '3rdparty') # could pick the wrong one. We also need to account for the fact that there # may be an executable 'wt' file the build directory and a subordinate .libs # directory. +env_builddir = os.getenv('WT_BUILDDIR') curdir = os.getcwd() -if os.path.basename(curdir) == '.libs' and \ +if env_builddir and os.path.isfile(os.path.join(env_builddir, 'wt')): + wt_builddir = env_builddir +elif os.path.basename(curdir) == '.libs' and \ os.path.isfile(os.path.join(curdir, os.pardir, 'wt')): wt_builddir = os.path.join(curdir, os.pardir) elif os.path.isfile(os.path.join(curdir, 'wt')): @@ -133,6 +136,7 @@ Options:\n\ -R | --randomseed run with random seeds for generates random numbers\n\ -S | --seed run with two seeds that generates random numbers, \n\ format "seed1.seed2", seed1 or seed2 can\'t be zero\n\ + -z | --zstd run the zstd tests\n\ \n\ Tests:\n\ may be a file name in test/suite: (e.g. test_base01.py)\n\ @@ -307,7 +311,7 @@ def error(exitval, prefix, msg): if __name__ == '__main__': # Turn numbers and ranges into test module names - preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = ignoreStdout = False + preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = zstdtest = ignoreStdout = False removeAtStart = True asan = False parallel = 0 @@ -380,6 +384,9 @@ if __name__ == '__main__': if option == '-long' or option == 'l': longtest = True continue + if option == '-zstd' or option == 'z': + zstdtest = True + continue if option == '-noremove': removeAtStart = False continue @@ -537,7 +544,7 @@ if __name__ == '__main__': # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. wttest.WiredTigerTestCase.globalSetup(preserve, removeAtStart, timestamp, gdbSub, lldbSub, - verbose, wt_builddir, dirarg, longtest, + verbose, wt_builddir, dirarg, longtest, zstdtest, ignoreStdout, seedw, seedz, hookmgr) # Without any tests listed as arguments, do discovery diff --git a/src/third_party/wiredtiger/test/suite/test_alter03.py b/src/third_party/wiredtiger/test/suite/test_alter03.py index 2ff8ef3e568..f517721b26f 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter03.py +++ b/src/third_party/wiredtiger/test/suite/test_alter03.py @@ -34,21 +34,27 @@ from wtscenario import make_scenarios class test_alter03(wttest.WiredTigerTestCase): name = "alter03" - def verify_metadata(self, table_metastr, file_metastr): - if table_metastr == '' and file_metastr == '': - return + def verify_metadata(self, table_metastr, lsm_metastr, file_metastr): c = self.session.open_cursor('metadata:', None, None) if table_metastr != '': - # We must find a table type entry for this object and it's value + # We must find a table type entry for this object and its value # should contain the provided table meta string. c.set_key('table:' + self.name) self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND) value = c.get_value() self.assertTrue(value.find(table_metastr) != -1) + if lsm_metastr != '': + # We must find a lsm type entry for this object and its value + # should contain the provided lsm meta string. + c.set_key('lsm:' + self.name) + self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND) + value = c.get_value() + self.assertTrue(value.find(lsm_metastr) != -1) + if file_metastr != '': - # We must find a file type entry for the object and it's value + # We must find a file type entry for the object and its value # should contain the provided file meta string. c.set_key('file:' + self.name + '.wt') self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND) @@ -73,20 +79,20 @@ class test_alter03(wttest.WiredTigerTestCase): c.close() # Verify the string in the metadata - self.verify_metadata(app_meta_orig, app_meta_orig) + self.verify_metadata(app_meta_orig, '', app_meta_orig) # Alter app metadata and verify self.session.alter(uri, 'app_metadata="meta_data_2",') - self.verify_metadata('app_metadata="meta_data_2",', 'app_metadata="meta_data_2",') + self.verify_metadata('app_metadata="meta_data_2",', '', 'app_metadata="meta_data_2",') # Alter app metadata, explicitly asking for exclusive access and verify self.session.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,') - self.verify_metadata('app_metadata="meta_data_3",', 'app_metadata="meta_data_3",') + self.verify_metadata('app_metadata="meta_data_3",', '', 'app_metadata="meta_data_3",') # Alter app metadata without taking exclusive lock and verify that only # table object gets modified self.session.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,') - self.verify_metadata('app_metadata="meta_data_4",', 'app_metadata="meta_data_3",') + self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') # Open a cursor, insert some data and try to alter with session open. # We should fail unless we ask not to take an exclusive lock @@ -96,22 +102,22 @@ class test_alter03(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: self.session.alter(uri, 'app_metadata="meta_data_5",')) - self.verify_metadata('app_metadata="meta_data_4",', 'app_metadata="meta_data_3",') + self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') self.assertRaisesException(wiredtiger.WiredTigerError, lambda: self.session.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_5",')) - self.verify_metadata('app_metadata="meta_data_4",', 'app_metadata="meta_data_3",') + self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') self.session.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,') - self.verify_metadata('app_metadata="meta_data_5",', 'app_metadata="meta_data_3",') + self.verify_metadata('app_metadata="meta_data_5",', '', 'app_metadata="meta_data_3",') c2.close() # Close and reopen the connection. # Confirm we retain the app_metadata as expected after reopen self.reopen_conn() - self.verify_metadata('app_metadata="meta_data_5",', 'app_metadata="meta_data_3",') + self.verify_metadata('app_metadata="meta_data_5",', '', 'app_metadata="meta_data_3",') # Alter LSM: A non exclusive alter should not be allowed def test_alter03_lsm_app_metadata(self): @@ -121,12 +127,20 @@ class test_alter03(wttest.WiredTigerTestCase): self.session.create(uri, create_params + app_meta_orig) + # Try to alter app metadata without exclusive access and verify self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.alter(uri, 'exclusive_refreshed=false,app_metadata="meta_data_2",'), '/is applicable only on simple tables/') + self.verify_metadata('', 'app_metadata="meta_data_1",', '') + # Alter app metadata, explicitly asking for exclusive access and verify self.session.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",') + self.verify_metadata('', 'app_metadata="meta_data_2",', '') + + # Alter app metadata and verify + self.session.alter(uri, 'app_metadata="meta_data_3",') + self.verify_metadata('', 'app_metadata="meta_data_3",', '') if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_compact01.py b/src/third_party/wiredtiger/test/suite/test_compact01.py index 0c6644a6a8a..b5a21ce96a6 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_compact01.py +++ b/src/third_party/wiredtiger/test/suite/test_compact01.py @@ -62,6 +62,12 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): # Test compaction. def test_compact(self): + # FIXME-WT-7187 + # This test is temporarily disabled for OS/X, it fails often, but not consistently. + import platform + if platform.system() == 'Darwin': + self.skipTest('Compaction tests skipped, as they fail on OS/X') + # Populate an object uri = self.type + self.name ds = self.dataset(self, uri, self.nentries - 1, config=self.config) diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index 54140b7019b..81d636eee6b 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -110,6 +110,12 @@ class test_compact02(wttest.WiredTigerTestCase): # Create a table, add keys with both big and small values. def test_compact02(self): + # FIXME-WT-7187 + # This test is temporarily disabled for OS/X, it fails, but not consistently. + import platform + if platform.system() == 'Darwin': + self.skipTest('Compaction tests skipped, as they fail on OS/X') + self.ConnectionOpen(self.cacheSize) mb = 1024 * 1024 diff --git a/src/third_party/wiredtiger/test/suite/test_compress02.py b/src/third_party/wiredtiger/test/suite/test_compress02.py new file mode 100644 index 00000000000..a88b8f54304 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_compress02.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch, os, shutil, threading, time +from wtthread import checkpoint_thread, op_thread +from helper import copy_wiredtiger_home +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios +from wiredtiger import stat + +# test_compress02.py +# This test checks that the compression level can be reconfigured after restart if +# we are using zstd as the block compressor. Tables created before reconfiguration +# will still use the previous compression level. +# + +def timestamp_str(t): + return '%x' % t +class test_compress02(wttest.WiredTigerTestCase): + + # Create a table. + uri = "table:test_compress02" + nrows = 1000 + + def conn_config(self): + config = 'builtin_extension_config={zstd={compression_level=6}},cache_size=10MB,log=(enabled=true)' + return config + + def large_updates(self, uri, value, ds, nrows): + # Update a large number of records. + session = self.session + cursor = session.open_cursor(uri) + for i in range(0, nrows): + session.begin_transaction() + cursor[ds.key(i)] = value + session.commit_transaction() + cursor.close() + + def check(self, check_value, uri, nrows): + session = self.session + session.begin_transaction() + cursor = session.open_cursor(uri) + count = 0 + for k, v in cursor: + self.assertEqual(v, check_value) + count += 1 + session.commit_transaction() + self.assertEqual(count, nrows) + + # Load the compression extension, skip the test if missing + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', 'zstd') + + @wttest.zstdtest('Skip zstd on pcc and zseries machines') + def test_compress02(self): + + ds = SimpleDataSet(self, self.uri, 0, key_format="S", value_format="S",config='block_compressor=zstd,log=(enabled=false)') + ds.populate() + valuea = "aaaaa" * 100 + + cursor = self.session.open_cursor(self.uri) + self.large_updates(self.uri, valuea, ds, self.nrows) + + self.check(valuea, self.uri, self.nrows) + self.session.checkpoint() + + #Simulate a crash by copying to a new directory(RESTART). + copy_wiredtiger_home(self, ".", "RESTART") + + # Close the connection and reopen it with a different zstd compression level configuration. + restart_config = 'builtin_extension_config={zstd={compression_level=9}},cache_size=10MB,log=(enabled=true)' + self.close_conn() + self.reopen_conn("RESTART", restart_config) + + # Open the new directory. + self.session = self.setUpSessionOpen(self.conn) + + # Check the table contains the last checkpointed value. + self.check(valuea, self.uri, self.nrows) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode09.py b/src/third_party/wiredtiger/test/suite/test_debug_mode09.py new file mode 100755 index 00000000000..a44e25ca7ce --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode09.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from wiredtiger import stat +import wttest + +# test_debug_mode09.py +# Test the debug mode setting for update_restore_evict. +# Force update restore eviction, whenever we evict a page. The debug mode +# is only effective on high cache pressure as WiredTiger can potentially decide +# to do an update restore evict on a page, when the cache pressure requirements are not met. +# This means setting eviction target low and cache size high. +class test_debug_mode09(wttest.WiredTigerTestCase): + conn_config = 'cache_size=10MB,statistics=(all),eviction_target=10' + uri = "table:test_debug_mode09" + + # Insert a bunch of data to trigger eviction + def trigger_eviction(self, uri): + cursor = self.session.open_cursor(uri) + for i in range(0, 20000): + self.session.begin_transaction() + cursor[i] = 'b' * 500 + self.session.commit_transaction() + + def test_update_restore_evict(self): + self.session.create(self.uri, 'key_format=i,value_format=S') + + self.trigger_eviction(self.uri) + + # Read the statistics of pages that have been update restored without update_restore + stat_cursor = self.session.open_cursor('statistics:') + pages_update_restored = stat_cursor[stat.conn.cache_write_restore][2] + stat_cursor.close() + self.assertGreater(pages_update_restored, 0) + + # Restart the connection with update restore evict config and a clean table + self.close_conn() + self.conn_config += ",debug_mode=(update_restore_evict=true)" + self.reopen_conn(".", self.conn_config) + self.session.drop(self.uri) + self.session.create(self.uri, 'key_format=i,value_format=S') + + self.trigger_eviction(self.uri) + + stat_cursor = self.session.open_cursor('statistics:') + forced_pages_update_restore = stat_cursor[stat.conn.cache_write_restore][2] + stat_cursor.close() + self.assertGreater(forced_pages_update_restore, pages_update_restored) diff --git a/src/third_party/wiredtiger/test/suite/test_hs21.py b/src/third_party/wiredtiger/test/suite/test_hs21.py index e2c8885661f..920dc41951b 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs21.py +++ b/src/third_party/wiredtiger/test/suite/test_hs21.py @@ -47,7 +47,7 @@ class test_hs21(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' file_name = 'test_hs21' numfiles = 10 - nrows = 10000 + nrows = 1000 def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records, we'll hang if the history store table isn't working. diff --git a/src/third_party/wiredtiger/test/suite/test_hs23.py b/src/third_party/wiredtiger/test/suite/test_hs23.py new file mode 100644 index 00000000000..cfc7ad70623 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs23.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest + +def timestamp_str(t): + return '%x' % t + +# test_hs23.py +# Test the case that we have update, out of order timestamp +# update, and update again in the same transaction +class test_hs23(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB' + session_config = 'isolation=snapshot' + + def test(self): + uri = 'table:test_hs23' + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri) + self.conn.set_timestamp( + 'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1)) + + value1 = 'a' + value2 = 'b' + value3 = 'c' + value4 = 'd' + value5 = 'e' + + # Insert a key. + self.session.begin_transaction() + cursor[str(0)] = value1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + + # Update at 10, update at 20, update at 15 (out of order), and + # update at 20 in the same transaction + self.session.begin_transaction() + cursor.set_key(str(0)) + cursor.set_value(value2) + self.session.timestamp_transaction( + 'commit_timestamp=' + timestamp_str(10)) + self.assertEquals(cursor.update(), 0) + + cursor.set_key(str(0)) + cursor.set_value(value3) + self.session.timestamp_transaction( + 'commit_timestamp=' + timestamp_str(20)) + self.assertEquals(cursor.update(), 0) + + cursor.set_key(str(0)) + cursor.set_value(value4) + self.session.timestamp_transaction( + 'commit_timestamp=' + timestamp_str(15)) + self.assertEquals(cursor.update(), 0) + + cursor.set_key(str(0)) + cursor.set_value(value5) + self.session.timestamp_transaction( + 'commit_timestamp=' + timestamp_str(20)) + self.assertEquals(cursor.update(), 0) + self.session.commit_transaction() + + # Do a checkpoint to trigger + # history store reconciliation. + self.session.checkpoint() + + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + + # Search the key to evict it. + self.session.begin_transaction() + self.assertEqual(evict_cursor[str(0)], value5) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Search the latest update + self.session.begin_transaction("read_timestamp=" + timestamp_str(20)) + self.assertEqual(cursor[str(0)], value5) + self.session.rollback_transaction() + + # Serarch the out of order timestamp update + self.session.begin_transaction("read_timestamp=" + timestamp_str(15)) + self.assertEqual(cursor[str(0)], value4) + self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py index 4b638a4015f..515e84388e5 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered02.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py @@ -30,23 +30,29 @@ import os, wiredtiger, wtscenario, wttest from wtdataset import SimpleDataSet # test_tiered02.py -# Test block-log-structured tree configuration options. +# Test tiered tree class test_tiered02(wttest.WiredTigerTestCase): K = 1024 M = 1024 * K G = 1024 * M + # TODO: tiered: change this to a table: URI, otherwise we are + # not using tiered files. uri = "file:test_tiered02" auth_token = "test_token" bucket = "mybucket" bucket_prefix = "pfx_" extension_name = "local_store" + prefix = "pfx-" def conn_config(self): os.makedirs(self.bucket, exist_ok=True) return \ - 'tiered_storage=(auth_token={},bucket={},bucket_prefix={},name={})'.format( \ - self.auth_token, self.bucket, self.bucket_prefix, self.extension_name) + 'statistics=(all),' + \ + 'tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.prefix + \ + 'name=%s)' % self.extension_name # Load the local store extension, but skip the test if it is missing. def conn_extensions(self, extlist): @@ -72,8 +78,7 @@ class test_tiered02(wttest.WiredTigerTestCase): # replaced with flush_tier. def test_tiered(self): self.flushed_objects = 0 - args = 'key_format=S,block_allocation=log-structured' - self.verbose(3, 'Test log-structured allocation with config: ' + args) + args = 'key_format=S' ds = SimpleDataSet(self, self.uri, 10, config=args) ds.populate() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py index 670b2e4202c..4c870e366c2 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered03.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py @@ -36,6 +36,9 @@ class test_tiered03(wttest.WiredTigerTestCase): K = 1024 M = 1024 * K G = 1024 * M + # TODO: tiered: change this to a table: URI, otherwise we are + # not using tiered files. The use of a second directory for + # sharing would probably need to be reworked. uri = 'file:test_tiered03' # Occasionally add a lot of records, so that merges (and bloom) happen. @@ -44,12 +47,30 @@ class test_tiered03(wttest.WiredTigerTestCase): scenarios = wtscenario.make_scenarios(record_count_scenarios, prune=100, prunelong=500) + auth_token = "test_token" + bucket = "mybucket" + bucket_prefix = "pfx_" + extension_name = "local_store" + prefix = "pfx-" + + def conn_config(self): + if not os.path.exists(self.bucket): + os.mkdir(self.bucket) + return \ + 'statistics=(all),' + \ + 'tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.prefix + \ + 'name=%s)' % self.extension_name + + # Load the local store extension, but skip the test if it is missing. + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('storage_sources', self.extension_name) + # Test sharing data between a primary and a secondary def test_sharing(self): - args = 'block_allocation=log-structured' - self.verbose(3, - 'Test log-structured allocation with config: ' + args + ' count: ' + str(self.nrecs)) - ds = SimpleDataSet(self, self.uri, 10, config=args) + ds = SimpleDataSet(self, self.uri, 10) ds.populate() ds.check() self.session.checkpoint() @@ -71,7 +92,7 @@ class test_tiered03(wttest.WiredTigerTestCase): ds.check_cursor(cursor2) cursor2.close() - newds = SimpleDataSet(self, self.uri, 10000, config=args) + newds = SimpleDataSet(self, self.uri, 10000) newds.populate() newds.check() self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py index 75d4fac1b19..26254f4d33d 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered04.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py @@ -35,7 +35,7 @@ StorageSource = wiredtiger.StorageSource # easy access to constants class test_tiered04(wttest.WiredTigerTestCase): # If the 'uri' changes all the other names must change with it. - fileuri = 'file:test_tiered04-0000000001.wtobj' + fileuri_base = 'file:test_tiered04-000000000' objuri = 'object:test_tiered04-0000000001.wtobj' tiereduri = "tiered:test_tiered04" uri = "table:test_tiered04" @@ -118,12 +118,15 @@ class test_tiered04(wttest.WiredTigerTestCase): self.pr("flush tier again") self.session.flush_tier(None) calls = self.get_stat(stat.conn.flush_tier, None) - self.assertEqual(calls, 2) + flush = 2 + self.assertEqual(calls, flush) obj = self.get_stat(stat.conn.tiered_object_size, None) self.assertEqual(obj, self.object_sys_val) + # As we flush each object, we are currently removing the file: object. So N + 1 exists. + fileuri = self.fileuri_base + str(flush + 1) + '.wtobj' self.check_metadata(self.tiereduri, intl_page) - self.check_metadata(self.fileuri, intl_page) + self.check_metadata(fileuri, intl_page) self.check_metadata(self.objuri, intl_page) #self.pr("verify stats") @@ -155,10 +158,12 @@ class test_tiered04(wttest.WiredTigerTestCase): config = 'tiered_storage=(local_retention=%d)' % new self.pr("reconfigure") self.conn.reconfigure(config) - self.session.flush_tier(None) retain = self.get_stat(stat.conn.tiered_retention, None) - calls = self.get_stat(stat.conn.flush_tier, None) self.assertEqual(retain, new) + self.pr("reconfigure flush_tier") + self.session.flush_tier(None) + self.pr("reconfigure get stat") + calls = self.get_stat(stat.conn.flush_tier, None) self.assertEqual(calls, 5) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py index 5cbfe4366c7..097af289a2b 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, wiredtiger, wttest +import os, time, wiredtiger, wttest from wiredtiger import stat StorageSource = wiredtiger.StorageSource # easy access to constants @@ -39,7 +39,7 @@ class test_tiered05(wttest.WiredTigerTestCase): bucket = "my_bucket" bucket_prefix = "my_prefix" extension_name = "local_store" - bucket = "./objects" + wait = 2 def conn_extensions(self, extlist): extlist.skip_if_missing = True @@ -49,7 +49,7 @@ class test_tiered05(wttest.WiredTigerTestCase): os.mkdir(self.bucket) return \ 'statistics=(fast),' + \ - 'tiered_manager=(wait=10),' + \ + 'tiered_manager=(wait=%d),' % self.wait + \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ @@ -59,6 +59,7 @@ class test_tiered05(wttest.WiredTigerTestCase): # Test calling the flush_tier API with a tiered manager. Should get an error. def test_tiered(self): self.session.create(self.uri, 'key_format=S') + time.sleep(self.wait) msg = "/storage manager thread is configured/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.assertEquals(self.session.flush_tier(None), 0), msg) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 87db069fefe..ab4d7689d3f 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -203,7 +203,7 @@ class WiredTigerTestCase(unittest.TestCase): @staticmethod def globalSetup(preserveFiles = False, removeAtStart = True, useTimestamp = False, gdbSub = False, lldbSub = False, verbose = 1, builddir = None, dirarg = None, - longtest = False, ignoreStdout = False, seedw = 0, seedz = 0, hookmgr = None): + longtest = False, zstdtest = False, ignoreStdout = False, seedw = 0, seedz = 0, hookmgr = None): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg if useTimestamp: @@ -219,6 +219,7 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._gdbSubprocess = gdbSub WiredTigerTestCase._lldbSubprocess = lldbSub WiredTigerTestCase._longtest = longtest + WiredTigerTestCase._zstdtest = zstdtest WiredTigerTestCase._verbose = verbose WiredTigerTestCase._ignoreStdout = ignoreStdout WiredTigerTestCase._dupout = os.dup(sys.stdout.fileno()) @@ -404,8 +405,19 @@ class WiredTigerTestCase(unittest.TestCase): def setUp(self): if not hasattr(self.__class__, 'wt_ntests'): self.__class__.wt_ntests = 0 + + # We want to have a unique execution directory name for each test. + # When a test fails, or with the -p option, we want to preserve the + # directory. For the non concurrent case, this is easy, we use the + # class name (like test_base02) and append a class-specific counter. + # This is short, somewhat descriptive, and unique. + # + # Class-specific counters won't work for the concurrent case. To + # get uniqueness we use a sanitized "short" identifier. Despite its name, + # this is not short (with scenarios at least), but it is descriptive + # and unique. if WiredTigerTestCase._concurrent: - self.testsubdir = self.shortid() + '.' + str(self.__class__.wt_ntests) + self.testsubdir = self.sanitized_shortid() else: self.testsubdir = self.className() + '.' + str(self.__class__.wt_ntests) self.testdir = os.path.join(WiredTigerTestCase._parentTestdir, self.testsubdir) @@ -749,9 +761,31 @@ class WiredTigerTestCase(unittest.TestCase): def shortid(self): return self.id().replace("__main__.","") + def sanitized_shortid(self): + """ + Return a name that is suitable for creating file system names. + In particular, names with scenarios look like + 'test_file.test_file.test_funcname(scen1.scen2.scen3)'. + So transform '(', but remove final ')'. + """ + return self.shortid().translate(str.maketrans('($[]/ ','______', ')')) + def className(self): return self.__class__.__name__ +def zstdtest(description): + """ + Used as a function decorator, for example, @wttest.zstdtest("description"). + The decorator indicates that this test function should only be included + when running the test suite with the --zstd option. + """ + def runit_decorator(func): + return func + if not WiredTigerTestCase._zstdtest: + return unittest.skip(description + ' (enable with --zstd)') + else: + return runit_decorator + def longtest(description): """ Used as a function decorator, for example, @wttest.longtest("description"). diff --git a/src/third_party/wiredtiger/test/syscall/syscall.py b/src/third_party/wiredtiger/test/syscall/syscall.py index 874ba546944..a57c6bb99c9 100755 --- a/src/third_party/wiredtiger/test/syscall/syscall.py +++ b/src/third_party/wiredtiger/test/syscall/syscall.py @@ -770,8 +770,26 @@ class SyscallCommand: def parse_args(self, argv): srcdir = os.path.join(self.disttop, 'test', 'syscall') self.exetopdir = os.path.join(self.builddir, 'test', 'syscall') - self.incdir1 = os.path.join(self.disttop, 'src', 'include') - self.incdir2 = self.builddir + self.incdirs = [] + + if self.disttop == self.builddir: + # CTest runs a copy of the script in the build directory, so the src include + # is a level above. + self.incdirs.append(os.path.join(self.disttop, '..', 'src', 'include')) + else: + self.incdirs.append(os.path.join(self.disttop, 'src', 'include')) + + if os.path.isfile(os.path.join(self.builddir, 'wiredtiger_config.h')) and \ + os.path.isfile(os.path.join(self.builddir, 'wiredtiger.h')): + # When building with autoconf, the generated includes will be at the top level + # of the build directory. + self.incdirs.append(self.builddir) + elif os.path.isfile(os.path.join(self.builddir, 'config', 'wiredtiger_config.h')) and \ + os.path.isfile(os.path.join(self.builddir, 'include', 'wiredtiger.h')): + # When building with CMake, the generated includes will be in the config and include + # sub-directories. + self.incdirs.append(os.path.join(self.builddir, 'config')) + self.incdirs.append(os.path.join(self.builddir, 'include')) ap = argparse.ArgumentParser('Syscall test runner') ap.add_argument('--systype', @@ -881,8 +899,8 @@ class SyscallCommand: with open(probe_c, "w") as f: f.write(program) ccargs = ['cc', '-o', probe_exe] - ccargs.append('-I' + self.incdir1) - ccargs.append('-I' + self.incdir2) + for inc in self.incdirs: + ccargs.append('-I' + inc) if self.args.systype == 'Linux': ccargs.append('-D_GNU_SOURCE') ccargs.append(probe_c) diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 674a868c35e..16f79426533 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -141,11 +141,17 @@ testutil_clean_work_dir(const char *dir) * Get the git top level directory and concatenate the build directory. */ void -testutil_build_dir(char *buf, int size) +testutil_build_dir(TEST_OPTS *opts, char *buf, int size) { FILE *fp; char *p; + /* If a build directory was manually given as an option we can directly return this instead. */ + if (opts->build_dir != NULL) { + strncpy(buf, opts->build_dir, (size_t)size); + return; + } + /* Get the git top level directory. */ #ifdef _WIN32 fp = _popen("git rev-parse --show-toplevel", "r"); diff --git a/src/third_party/wiredtiger/test/utility/parse_opts.c b/src/third_party/wiredtiger/test/utility/parse_opts.c index 35b9e2a5542..8e5bf425db1 100644 --- a/src/third_party/wiredtiger/test/utility/parse_opts.c +++ b/src/third_party/wiredtiger/test/utility/parse_opts.c @@ -49,7 +49,7 @@ testutil_parse_opts(int argc, char *const *argv, TEST_OPTS *opts) testutil_print_command_line(argc, argv); - while ((ch = __wt_getopt(opts->progname, argc, argv, "A:dh:n:o:pR:T:t:vW:")) != EOF) + while ((ch = __wt_getopt(opts->progname, argc, argv, "A:dh:b:n:o:pR:T:t:vW:")) != EOF) switch (ch) { case 'A': /* Number of append threads */ opts->n_append_threads = (uint64_t)atoll(__wt_optarg); @@ -60,6 +60,9 @@ testutil_parse_opts(int argc, char *const *argv, TEST_OPTS *opts) case 'h': /* Home directory */ opts->home = dstrdup(__wt_optarg); break; + case 'b': /* Build directory */ + opts->build_dir = dstrdup(__wt_optarg); + break; case 'n': /* Number of records */ opts->nrecords = (uint64_t)atoll(__wt_optarg); break; @@ -104,6 +107,7 @@ testutil_parse_opts(int argc, char *const *argv, TEST_OPTS *opts) "[-A append thread count] " "[-d add data] " "[-h home] " + "[-b build directory] " "[-n record count] " "[-o op count] " "[-p] " diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index b657d5717a7..6689227d359 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -54,6 +54,7 @@ typedef struct { char *home; const char *argv0; /* Exec name */ const char *progname; /* Truncated program name */ + char *build_dir; /* Build directory path */ enum { TABLE_COL = 1, /* Fixed-length column store */ @@ -268,7 +269,7 @@ void testutil_clean_work_dir(const char *); void testutil_cleanup(TEST_OPTS *); void testutil_copy_data(const char *); bool testutil_is_flag_set(const char *); -void testutil_build_dir(char *, int); +void testutil_build_dir(TEST_OPTS *, char *, int); void testutil_make_work_dir(const char *); int testutil_parse_opts(int, char *const *, TEST_OPTS *); void testutil_print_command_line(int argc, char *const *argv); |