From 75696046467df0ef88c9034e0f4af31503717450 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Thu, 19 Aug 2021 14:26:05 +1000 Subject: Import wiredtiger: b385c984870fec2d693dece8e79876fd5e8bf867 from branch mongodb-5.0 ref: ffebd365d6..b385c98487 for: 5.0.3 WT-6915 Write "log files" subpage for Architecture Guide WT-7363 Add support for dumping history store output in hex format WT-7663 Change local store extension to allow only readonly FS operations WT-7718 Rename 'build_cmake' WT-7866 Update cache_hs_insert limits in cppsuite-hs-cleanup-stress WT-7880 Fix history store record issue when the update following the prepared update is in history store WT-7891 Remove doc typos WT-7907 Add dependencies to swig module definition in CMake build WT-7918 Support setting a prepare timestamp at current read timestamp WT-7928 VLCS checkpoint and additional test suite improvements WT-7934 Upload perf results from many-collection test to Altas WT-7936 Update the format.sh script to run recovery tests WT-7940 Update mongod path for many-coll test WT-7943 Do not assert timestamps when rolling back a prepared transactions WT-7945 Move rollback handling to the operation layer in the cppsuite. WT-7947 Allow CMake to take in a specific Python version WT-7952 Minor docs build fixes WT-7953 Teach s_string to not look inside getopt option lists. WT-7955 Copy format.sh and CONFIG.stress to the test/format build directory with CMake WT-7956 RTS to skip deleted or stable RLE cells WT-7961 Sometimes lag oldest timestamp in timestamp_abort. --- src/third_party/wiredtiger/build_cmake/README.md | 157 ----- .../build_cmake/configs/aarch64/linux/config.cmake | 34 -- .../wiredtiger/build_cmake/configs/auto.cmake | 325 ---------- .../wiredtiger/build_cmake/configs/base.cmake | 231 -------- .../compile_test/pthread_cond_monotonic_test.c | 40 -- .../wiredtiger/build_cmake/configs/modes.cmake | 112 ---- .../build_cmake/configs/ppc64le/linux/config.cmake | 27 - .../build_cmake/configs/s390x/linux/config.cmake | 22 - .../build_cmake/configs/wiredtiger_config.h.in | 177 ------ .../build_cmake/configs/x86/darwin/config.cmake | 11 - .../build_cmake/configs/x86/linux/config.cmake | 19 - .../build_cmake/configs/x86/netbsd/config.cmake | 14 - .../build_cmake/configs/x86/windows/config.cmake | 40 -- .../wiredtiger/build_cmake/helpers.cmake | 652 --------------------- .../wiredtiger/build_cmake/install/install.cmake | 37 -- .../build_cmake/install/wiredtiger.pc.in | 11 - .../wiredtiger/build_cmake/strict/cl_strict.cmake | 21 - .../build_cmake/strict/clang_strict.cmake | 65 -- .../wiredtiger/build_cmake/strict/gcc_strict.cmake | 79 --- .../toolchains/aarch64/linux/plat_clang.cmake | 15 - .../toolchains/aarch64/linux/plat_gcc.cmake | 11 - .../wiredtiger/build_cmake/toolchains/cl.cmake | 13 - .../wiredtiger/build_cmake/toolchains/clang.cmake | 36 -- .../wiredtiger/build_cmake/toolchains/gcc.cmake | 31 - .../toolchains/mongodbtoolchain_v3_clang.cmake | 17 - .../toolchains/mongodbtoolchain_v3_gcc.cmake | 17 - .../toolchains/ppc64le/linux/plat_clang.cmake | 15 - .../toolchains/ppc64le/linux/plat_gcc.cmake | 11 - .../toolchains/s390x/linux/plat_clang.cmake | 15 - .../toolchains/s390x/linux/plat_gcc.cmake | 11 - .../toolchains/x86/darwin/plat_clang.cmake | 15 - .../toolchains/x86/darwin/plat_gcc.cmake | 14 - .../toolchains/x86/linux/plat_clang.cmake | 15 - .../toolchains/x86/linux/plat_gcc.cmake | 11 - src/third_party/wiredtiger/cmake/README.md | 157 +++++ .../cmake/configs/aarch64/linux/config.cmake | 34 ++ .../wiredtiger/cmake/configs/auto.cmake | 325 ++++++++++ .../wiredtiger/cmake/configs/base.cmake | 241 ++++++++ .../compile_test/pthread_cond_monotonic_test.c | 40 ++ .../wiredtiger/cmake/configs/modes.cmake | 112 ++++ .../cmake/configs/ppc64le/linux/config.cmake | 27 + .../cmake/configs/s390x/linux/config.cmake | 22 + .../cmake/configs/wiredtiger_config.h.in | 177 ++++++ .../cmake/configs/x86/darwin/config.cmake | 11 + .../cmake/configs/x86/linux/config.cmake | 19 + .../cmake/configs/x86/netbsd/config.cmake | 14 + .../cmake/configs/x86/windows/config.cmake | 40 ++ src/third_party/wiredtiger/cmake/helpers.cmake | 648 ++++++++++++++++++++ .../wiredtiger/cmake/install/install.cmake | 37 ++ .../wiredtiger/cmake/install/wiredtiger.pc.in | 11 + .../wiredtiger/cmake/strict/cl_strict.cmake | 21 + .../wiredtiger/cmake/strict/clang_strict.cmake | 65 ++ .../wiredtiger/cmake/strict/gcc_strict.cmake | 79 +++ .../toolchains/aarch64/linux/plat_clang.cmake | 15 + .../cmake/toolchains/aarch64/linux/plat_gcc.cmake | 11 + .../wiredtiger/cmake/toolchains/cl.cmake | 13 + .../wiredtiger/cmake/toolchains/clang.cmake | 36 ++ .../wiredtiger/cmake/toolchains/gcc.cmake | 31 + .../toolchains/mongodbtoolchain_v3_clang.cmake | 17 + .../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake | 17 + .../toolchains/ppc64le/linux/plat_clang.cmake | 15 + .../cmake/toolchains/ppc64le/linux/plat_gcc.cmake | 11 + .../cmake/toolchains/s390x/linux/plat_clang.cmake | 15 + .../cmake/toolchains/s390x/linux/plat_gcc.cmake | 11 + .../cmake/toolchains/x86/darwin/plat_clang.cmake | 15 + .../cmake/toolchains/x86/darwin/plat_gcc.cmake | 14 + .../cmake/toolchains/x86/linux/plat_clang.cmake | 15 + .../cmake/toolchains/x86/linux/plat_gcc.cmake | 11 + src/third_party/wiredtiger/dist/api_data.py | 14 +- src/third_party/wiredtiger/dist/s_copyright.list | 2 +- src/third_party/wiredtiger/dist/s_docs | 2 +- src/third_party/wiredtiger/dist/s_export | 2 +- src/third_party/wiredtiger/dist/s_string | 3 +- src/third_party/wiredtiger/dist/s_string.ok | 15 - src/third_party/wiredtiger/dist/stat_data.py | 2 + .../ext/storage_sources/local_store/local_store.c | 296 ++++------ src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/btree/bt_curnext.c | 4 +- src/third_party/wiredtiger/src/btree/bt_cursor.c | 2 +- src/third_party/wiredtiger/src/btree/bt_debug.c | 4 +- src/third_party/wiredtiger/src/btree/bt_misc.c | 4 +- src/third_party/wiredtiger/src/btree/bt_slvg.c | 9 +- src/third_party/wiredtiger/src/btree/bt_vrfy.c | 11 +- src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c | 5 +- src/third_party/wiredtiger/src/btree/col_modify.c | 50 +- src/third_party/wiredtiger/src/btree/row_modify.c | 23 +- src/third_party/wiredtiger/src/config/config_def.c | 5 +- src/third_party/wiredtiger/src/cursor/cur_bulk.c | 4 +- src/third_party/wiredtiger/src/cursor/cur_dump.c | 8 +- src/third_party/wiredtiger/src/cursor/cur_std.c | 2 + src/third_party/wiredtiger/src/docs/arch-hs.dox | 2 +- .../wiredtiger/src/docs/arch-log-file.dox | 60 +- .../wiredtiger/src/docs/build-posix.dox | 6 +- .../wiredtiger/src/docs/command-line.dox | 8 +- .../wiredtiger/src/docs/dump-formats.dox | 11 +- src/third_party/wiredtiger/src/docs/schema.dox | 2 +- .../wiredtiger/src/docs/transactions.dox | 9 + src/third_party/wiredtiger/src/docs/upgrading.dox | 2 +- src/third_party/wiredtiger/src/history/hs_verify.c | 4 +- src/third_party/wiredtiger/src/include/extern.h | 9 +- .../wiredtiger/src/include/serial_inline.h | 2 +- src/third_party/wiredtiger/src/include/stat.h | 4 + .../wiredtiger/src/include/txn_inline.h | 6 +- .../wiredtiger/src/include/wiredtiger.in | 141 +++-- src/third_party/wiredtiger/src/reconcile/rec_col.c | 4 +- .../wiredtiger/src/reconcile/rec_write.c | 2 +- src/third_party/wiredtiger/src/support/scratch.c | 20 +- src/third_party/wiredtiger/src/support/stat.c | 14 + .../wiredtiger/src/tiered/tiered_opener.c | 6 +- src/third_party/wiredtiger/src/txn/txn.c | 58 +- .../wiredtiger/src/txn/txn_rollback_to_stable.c | 30 +- src/third_party/wiredtiger/src/txn/txn_timestamp.c | 14 +- .../wiredtiger/src/utilities/util_dump.c | 65 +- .../wiredtiger/test/checkpoint/checkpointer.c | 1 - .../wiredtiger/test/checkpoint/smoke.sh | 56 +- .../wiredtiger/test/checkpoint/test_checkpoint.c | 5 +- .../wiredtiger/test/checkpoint/workers.c | 3 +- .../test/cppsuite/configs/hs_cleanup_stress.txt | 4 +- .../wiredtiger/test/cppsuite/test_harness/test.cxx | 4 +- .../cppsuite/test_harness/timestamp_manager.cxx | 12 +- .../test_harness/workload/database_operation.cxx | 66 ++- .../test_harness/workload/thread_context.cxx | 87 ++- .../test_harness/workload/thread_context.h | 36 +- .../wiredtiger/test/cppsuite/tests/hs_cleanup.cxx | 31 +- .../wiredtiger/test/csuite/random_abort/main.c | 14 +- .../wiredtiger/test/csuite/timestamp_abort/main.c | 16 +- .../wiredtiger/test/cursor_order/Makefile.am | 2 +- .../wiredtiger/test/cursor_order/smoke.sh | 14 + src/third_party/wiredtiger/test/evergreen.yml | 23 +- .../wiredtiger/test/evergreen/build_windows.ps1 | 2 +- src/third_party/wiredtiger/test/format/format.sh | 54 +- src/third_party/wiredtiger/test/format/recover.sh | 52 -- .../wiredtiger/test/suite/test_assert06.py | 11 +- .../wiredtiger/test/suite/test_bug022.py | 20 +- .../wiredtiger/test/suite/test_dump01.py | 131 +++++ .../wiredtiger/test/suite/test_durable_ts03.py | 15 +- .../wiredtiger/test/suite/test_prepare06.py | 20 +- .../wiredtiger/test/suite/test_tiered06.py | 149 ++--- .../wiredtiger/test/suite/test_timestamp10.py | 15 +- .../wiredtiger/test/suite/test_timestamp11.py | 44 +- .../wiredtiger/test/suite/test_timestamp12.py | 13 +- .../wiredtiger/test/suite/test_timestamp14.py | 22 +- .../wiredtiger/test/suite/test_timestamp16.py | 1 - .../wiredtiger/test/suite/test_timestamp17.py | 8 +- .../wiredtiger/test/suite/test_timestamp18.py | 32 +- .../wiredtiger/test/suite/test_timestamp19.py | 13 +- .../wiredtiger/test/suite/test_timestamp20.py | 44 +- .../wiredtiger/test/suite/test_timestamp22.py | 17 +- .../wiredtiger/test/suite/test_timestamp23.py | 123 ++++ src/third_party/wiredtiger/test/test_coverage.md | 2 +- 150 files changed, 3642 insertions(+), 3116 deletions(-) delete mode 100644 src/third_party/wiredtiger/build_cmake/README.md delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/auto.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/base.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/compile_test/pthread_cond_monotonic_test.c delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/modes.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/x86/darwin/config.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/x86/netbsd/config.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/configs/x86/windows/config.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/helpers.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/install/install.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/install/wiredtiger.pc.in delete mode 100644 src/third_party/wiredtiger/build_cmake/strict/cl_strict.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/cl.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_clang.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake delete mode 100644 src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake create mode 100644 src/third_party/wiredtiger/cmake/README.md create mode 100644 src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/auto.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/base.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/compile_test/pthread_cond_monotonic_test.c create mode 100644 src/third_party/wiredtiger/cmake/configs/modes.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/ppc64le/linux/config.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/s390x/linux/config.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in create mode 100644 src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/x86/netbsd/config.cmake create mode 100644 src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake create mode 100644 src/third_party/wiredtiger/cmake/helpers.cmake create mode 100644 src/third_party/wiredtiger/cmake/install/install.cmake create mode 100644 src/third_party/wiredtiger/cmake/install/wiredtiger.pc.in create mode 100644 src/third_party/wiredtiger/cmake/strict/cl_strict.cmake create mode 100644 src/third_party/wiredtiger/cmake/strict/clang_strict.cmake create mode 100644 src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_clang.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_gcc.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/cl.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/clang.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/gcc.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_clang.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_gcc.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_clang.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_gcc.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_clang.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_gcc.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_clang.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_gcc.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_clang.cmake create mode 100644 src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_gcc.cmake create mode 100755 src/third_party/wiredtiger/test/cursor_order/smoke.sh delete mode 100644 src/third_party/wiredtiger/test/format/recover.sh create mode 100644 src/third_party/wiredtiger/test/suite/test_dump01.py create mode 100644 src/third_party/wiredtiger/test/suite/test_timestamp23.py diff --git a/src/third_party/wiredtiger/build_cmake/README.md b/src/third_party/wiredtiger/build_cmake/README.md deleted file mode 100644 index caea84aec53..00000000000 --- a/src/third_party/wiredtiger/build_cmake/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# Building WiredTiger with CMake - -### Build Dependencies - -To build with CMake we **require** the following dependencies: - -* `cmake` : Official CMake install instructions found here: https://cmake.org/install/ - * *WiredTiger supports CMake 3.10+* -* `ninja` : Official ninja install instructions found here: https://ninja-build.org/ -* `python3-dev` : Consult your package management application for installation instructions. This is needed for building with Python support. -* `swig`: Consult your package management application for installation instructions. This is needed for building with Python support. - -We also strongly suggest the following dependencies are also installed (for improved build times): - -* `ccache` : Official ccache download instructions found here: https://ccache.dev/download.html - -##### Package Manager Instructions - -Alternatively you can use your system's package manager to install the dependencies listed above. Depending on the system, the following commands can be run: - -###### Install commands for Ubuntu & Debian (tested on Ubuntu 18.04) - -```bash -sudo apt-get install cmake cmake-curses-gui -sudo apt-get install ccache -sudo apt-get install ninja-build -sudo apt-get install python3-dev swig -``` - -###### Install commands for Mac (using HomeBrew) - -```bash -brew install ninja -brew install ccache -brew install cmake -brew install python -brew install swig -``` - -###### Install commands for Windows (using Chocolatey) - -```bash -choco install cmake -choco install ninja -choco install ccache --version=3.7.9 -choco install swig -choco install python --pre -``` - -### Building the WiredTiger Library - -> *The below commands are written for Linux and Darwin hosts. Windows instructions coming soon!* - -Building the WiredTiger library is relatively straightforward. Navigate to the top level of the WiredTiger repository and run the following commands: - -###### Configure your build - -```bash -# Create a new directory to run your build from -$ mkdir build && cd build -# Run the cmake configure step. Note: '-G Ninja' tells CMake to generate a ninja build -$ cmake -G Ninja ../. -... --- Configuring done --- Generating done --- Build files have been written to: /home/wiredtiger/build -``` - -*See [Configuration Options](#configuration-options) for additional configuration options.* - -###### Run your build - -In the same directory you configured your build, run the `ninja` command to start the build: - -```bash -$ ninja -... -[211/211 (100%) 2.464s] Creating library symlink libwiredtiger.so -``` - -*Note: Ninja doesn't need a `-j` option; it knows how many cores are available.* - -###### Configuration Options - -There are a number of additional configuration options you can pass to the CMake configuration step. A summary of some important options you will come to know: - -* `-DENABLE_STATIC=1` : Compile WiredTiger as a static library -* `-DENABLE_LZ4=1` : Build the lz4 compressor extension -* `-DENABLE_SNAPPY=1` : Build the snappy compressor extension -* `-DENABLE_ZLIB=1` : Build the zlib compressor extension -* `-DENABLE_ZSTD=1` : Build the libzstd compressor extension -* `-DENABLE_SODIUM=1` : Build the libsodium encryptor extension -* `-DHAVE_DIAGNOSTIC=1` : Enable WiredTiger diagnostics -* `-DHAVE_ATTACH=1` : Enable to pause for debugger attach on failure -* `-DENABLE_STRICT=1` : Compile with strict compiler warnings enabled -* `-DENABLE_PYTHON=1` : Compile the python API -* `-DCMAKE_INSTALL_PREFIX=` : Path to install directory - ---- - -An example of using the above configuration options during the configuration step: - -```bash -$ cmake -DENABLE_STATIC=1 -DENABLE_LZ4=1 -DENABLE_SNAPPY=1 -DENABLE_ZLIB=1 -DENABLE_ZSTD=1 -DHAVE_DIAGNOSTIC=1 -DHAVE_ATTACH=1 -DENABLE_STRICT=1 -G Ninja ../. -``` - ---- - -You can further look at all the available configuration options (and also dynamically change them!) by running `ccmake` in your build directory: - -```bash -$ cd build -$ ccmake . -``` - -*The configuration options can also be viewed in `build_cmake/configs/base.cmake`*. - -###### Switching between GCC and Clang (POSIX only) - -By default CMake will use your default system compiler (`cc`). If you want to use a specific toolchain you can pass a toolchain file! We have provided a toolchain file for both GCC (`build_cmake/toolchains/gcc.cmake`) and Clang (`build_cmake/toolchains/clang.cmake`). To use either toolchain you can pass the `-DCMAKE_TOOLCHAIN_FILE=` to the CMake configuration step. For example: - -*Using the GCC Toolchain* - -```bash -$ cd build -$ cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/gcc.cmake -G Ninja ../. -``` - -*Using the Clang Toolchain* - -```bash -$ cd build -$ cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/clang.cmake -G Ninja ../. -``` - -### Running WiredTiger C Tests - -The WiredTiger CMake build makes available a suite of C/C++ based tests (separate from the Python testsuite). The run all the tests available, ensure you're in the build directory and execute: - -```bash -ctest -j$(nproc) -``` - -To run with verbose output: - -```bash -ctest -j$(nproc) -VV -``` - -To run a specfic test: - -```bash -# Note: -R specifies a regex, where any matching test will be run -ctest -R -j$(nproc) -``` - -See `ctest --help` for a range of additional supported options. diff --git a/src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake deleted file mode 100644 index 94447ac9ca8..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/aarch64/linux/config.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# -include(CheckCCompilerFlag) - -set(WT_ARCH "aarch64" CACHE STRING "") -set(WT_OS "linux" CACHE STRING "") -set(WT_POSIX ON CACHE BOOL "") - -# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions -# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based -# builds. -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) - -# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. -set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") - -# ARMv8-A is the 64-bit ARM architecture, turn on the optional CRC instructions. -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc" CACHE STRING "" FORCE) - -check_c_compiler_flag("-moutline-atomics" has_moutline_atomics) - -# moutline-atomics preserves backwards compatibility with Arm v8.0 systems but also supports -# using Arm v8.1 atomics. The latter can massively improve performance on larger Arm systems. -# The flag was back ported to gcc8, 9 and is the default in gcc10+. See if the compiler supports -# the flag. -if(has_moutline_atomics) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -moutline-atomics" CACHE STRING "" FORCE) -endif() -unset(has_moutline_atomics CACHE) diff --git a/src/third_party/wiredtiger/build_cmake/configs/auto.cmake b/src/third_party/wiredtiger/build_cmake/configs/auto.cmake deleted file mode 100644 index b9026d90d70..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/auto.cmake +++ /dev/null @@ -1,325 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -include(build_cmake/helpers.cmake) - -### Auto configure options and checks that we can infer from our toolchain environment. - -## Assert type sizes. -assert_type_size("size_t" 8) -assert_type_size("ssize_t" 8) -assert_type_size("time_t" 8) -assert_type_size("off_t" 0) -assert_type_size("uintptr_t" 0) -test_type_size("uintmax_t" u_intmax_size) -test_type_size("unsigned long long" u_long_long_size) -set(default_uintmax_def " ") -if(${u_intmax_size} STREQUAL "") - if(${unsigned long long} STREQUAL "") - set(default_uintmax_def "typedef unsigned long uintmax_t\\;") - else() - set(default_uintmax_def "typedef unsigned long long uintmax_t\\;") - endif() -endif() - -set(default_offt_def) -if("${WT_OS}" STREQUAL "windows") - set(default_offt_def "typedef int64_t wt_off_t\\;") -else() - set(default_offt_def "typedef off_t wt_off_t\\;") -endif() - -config_string( - off_t_decl - "off_t type declaration." - DEFAULT "${default_offt_def}" - INTERNAL -) - -config_string( - uintprt_t_decl - "uintptr_t type declaration." - DEFAULT "${default_uintmax_def}" - INTERNAL -) - -config_include( - HAVE_SYS_TYPES_H - "Include header sys/types.h exists." - FILE "sys/types.h" -) - -config_include( - HAVE_INTTYPES_H - "Include header inttypes.h exists." - FILE "inttypes.h" -) - -config_include( - HAVE_STDARG_H - "Include header stdarg.h exists." - FILE "stdarg.h" -) - -config_include( - HAVE_STDBOOL_H - "Include header stdbool.h exists." - FILE "stdbool.h" -) - -config_include( - HAVE_STDINT_H - "Include header stdint.h exists." - FILE "stdint.h" -) - -config_include( - HAVE_STDLIB_H - "Include header stdlib.h exists." - FILE "stdlib.h" -) - -config_include( - HAVE_STDIO_H - "Include header stdio.h exists." - FILE "stdio.h" -) - -config_include( - HAVE_STRINGS_H - "Include header strings.h exists." - FILE "strings.h" -) - -config_include( - HAVE_STRING_H - "Include header string.h exists." - FILE "string.h" -) - -config_include( - HAVE_SYS_STAT_H - "Include header sys/stat.h exists." - FILE "sys/stat.h" -) - -config_include( - HAVE_UNISTD_H - "Include header unistd.h exists." - FILE "unistd.h" -) - -config_include( - HAVE_X86INTRIN_H - "Include header x86intrin.h exists." - FILE "x86intrin.h" -) - -config_include( - HAVE_DLFCN_H - "Include header dlfcn.h exists." - FILE "dlfcn.h" -) - -config_include( - HAVE_MEMORY_H - "Include header memory.h exists." - FILE "memory.h" -) - -config_func( - HAVE_CLOCK_GETTIME - "Function clock_gettime exists." - FUNC "clock_gettime" - FILES "time.h" -) - -config_func( - HAVE_FALLOCATE - "Function fallocate exists." - FUNC "fallocate" - FILES "fcntl.h" -) - -config_func( - HAVE_FDATASYNC - "Function fdatasync exists." - FUNC "fdatasync" - FILES "unistd.h" - DEPENDS "NOT WT_DARWIN" -) - -config_func( - HAVE_FTRUNCATE - "Function ftruncate exists." - FUNC "ftruncate" - FILES "unistd.h;sys/types.h" -) - -config_func( - HAVE_GETTIMEOFDAY - "Function gettimeofday exists." - FUNC "gettimeofday" - FILES "sys/time.h" -) - -config_func( - HAVE_POSIX_FADVISE - "Function posix_fadvise exists." - FUNC "posix_fadvise" - FILES "fcntl.h" -) - -config_func( - HAVE_POSIX_FALLOCATE - "Function posix_fallocate exists." - FUNC "posix_fallocate" - FILES "fcntl.h" -) - -config_func( - HAVE_POSIX_MADVISE - "Function posix_madvise exists." - FUNC "posix_madvise" - FILES "sys/mman.h" -) - -config_func( - HAVE_POSIX_MEMALIGN - "Function posix_memalign exists." - FUNC "posix_memalign" - FILES "stdlib.h" -) - -config_func( - HAVE_SETRLIMIT - "Function setrlimit exists." - FUNC "setrlimit" - FILES "sys/time.h;sys/resource.h" -) - -config_func( - HAVE_STRTOUQ - "Function strtouq exists." - FUNC "strtouq" - FILES "stdlib.h" -) - -config_func( - HAVE_SYNC_FILE_RANGE - "Function sync_file_range exists." - FUNC "sync_file_range" - FILES "fcntl.h" -) - -config_func( - HAVE_TIMER_CREATE - "Function timer_create exists." - FUNC "timer_create" - FILES "signal.h;time.h" - LIBS "rt" -) - -config_lib( - HAVE_LIBPTHREAD - "Pthread library exists." - LIB "pthread" -) - -config_lib( - HAVE_LIBRT - "rt library exists." - LIB "rt" -) - -config_lib( - HAVE_LIBDL - "dl library exists." - LIB "dl" -) - -config_lib( - HAVE_LIBLZ4 - "lz4 library exists." - LIB "lz4" - HEADER "lz4.h" -) - -config_lib( - HAVE_LIBSNAPPY - "snappy library exists." - LIB "snappy" - HEADER "snappy.h" -) - -config_lib( - HAVE_LIBZ - "zlib library exists." - LIB "z" - HEADER "zlib.h" -) - -config_lib( - HAVE_LIBZSTD - "zstd library exists." - LIB "zstd" - HEADER "zstd.h" -) - -config_lib( - HAVE_LIBSODIUM - "sodium library exists." - LIB "sodium" - HEADER "sodium.h" -) - -config_lib( - HAVE_LIBTCMALLOC - "tcmalloc library exists." - LIB "tcmalloc" -) - -config_compile( - HAVE_PTHREAD_COND_MONOTONIC - "If pthread condition variables support monotonic clocks." - SOURCE "${CMAKE_CURRENT_LIST_DIR}/compile_test/pthread_cond_monotonic_test.c" - LIBS "pthread" - DEPENDS "HAVE_LIBPTHREAD" -) - -include(TestBigEndian) -test_big_endian(is_big_endian) -if(NOT is_big_endian) - set(is_big_endian FALSE) -endif() -config_bool( - WORDS_BIGENDIAN - "If the target system is big endian" - DEFAULT ${is_big_endian} -) - -set(wiredtiger_includes_decl) -if(HAVE_SYS_TYPES_H) - list(APPEND wiredtiger_includes_decl "#include ") -endif() -if(HAVE_INTTYPES_H AND (NOT "${WT_OS}" STREQUAL "windows")) - list(APPEND wiredtiger_includes_decl "#include ") -endif() -if(HAVE_STDARG_H) - list(APPEND wiredtiger_includes_decl "#include ") -endif() -if(HAVE_STDBOOL_H) - list(APPEND wiredtiger_includes_decl "#include ") -endif() -if(HAVE_STDINT_H) - list(APPEND wiredtiger_includes_decl "#include ") -endif() -if(HAVE_STDIO_H) - list(APPEND wiredtiger_includes_decl "#include ") -endif() -string(REGEX REPLACE ";" "\n" wiredtiger_includes_decl "${wiredtiger_includes_decl}") diff --git a/src/third_party/wiredtiger/build_cmake/configs/base.cmake b/src/third_party/wiredtiger/build_cmake/configs/base.cmake deleted file mode 100644 index 2b44a7c85cb..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/base.cmake +++ /dev/null @@ -1,231 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -include(build_cmake/helpers.cmake) - -# WiredTiger-related configuration options. - -config_choice( - WT_ARCH - "Target architecture for WiredTiger" - OPTIONS - "x86;WT_X86;" - "aarch64;WT_AARCH64;" - "ppc64le;WT_PPC64;" - "s390x;WT_S390X;" -) - -config_choice( - WT_OS - "Target OS for WiredTiger" - OPTIONS - "darwin;WT_DARWIN;" - "windows;WT_WIN;" - "linux;WT_LINUX;" -) - -config_bool( - WT_POSIX - "Is a posix platform" - DEFAULT ON - DEPENDS "WT_LINUX OR WT_DARWIN" -) - -config_string( - WT_BUFFER_ALIGNMENT_DEFAULT - "WiredTiger buffer boundary aligment" - DEFAULT 0 -) - -config_bool( - HAVE_DIAGNOSTIC - "Enable WiredTiger diagnostics" - DEFAULT OFF -) - -config_bool( - HAVE_ATTACH - "Enable to pause for debugger attach on failure" - DEFAULT OFF -) - -config_bool( - ENABLE_STATIC - "Compile as a static library" - DEFAULT OFF -) - -config_bool( - ENABLE_STRICT - "Compile with strict compiler warnings enabled" - DEFAULT OFF -) - -config_bool( - ENABLE_PYTHON - "Configure the python API" - DEFAULT OFF -) - -config_bool( - WT_STANDALONE_BUILD - "Support standalone build" - DEFAULT ON -) - -config_bool( - HAVE_NO_CRC32_HARDWARE - "Disable any crc32 hardware support" - DEFAULT OFF -) - -config_bool( - DYNAMIC_CRT - "Link with the MSVCRT DLL version" - DEFAULT OFF - DEPENDS "WT_WIN" -) - -config_choice( - SPINLOCK_TYPE - "Set a spinlock type" - OPTIONS - "pthread;SPINLOCK_PTHREAD_MUTEX;HAVE_LIBPTHREAD" - "gcc;SPINLOCK_GCC;" - "msvc;SPINLOCK_MSVC;WT_WIN" - "pthread_adaptive;SPINLOCK_PTHREAD_ADAPTIVE;HAVE_LIBPTHREAD" -) - -config_bool( - ENABLE_LZ4 - "Build the lz4 compressor extension" - DEFAULT OFF - DEPENDS "HAVE_LIBLZ4" - # Specifically throw a fatal error if a user tries to enable the lz4 compressor without - # actually having the library available (as opposed to silently defaulting to OFF). - DEPENDS_ERROR ON "Failed to find lz4 library" -) - -config_bool( - ENABLE_SNAPPY - "Build the snappy compressor extension" - DEFAULT OFF - DEPENDS "HAVE_LIBSNAPPY" - # Specifically throw a fatal error if a user tries to enable the snappy compressor without - # actually having the library available (as opposed to silently defaulting to OFF). - DEPENDS_ERROR ON "Failed to find snappy library" -) - -config_bool( - ENABLE_ZLIB - "Build the zlib compressor extension" - DEFAULT OFF - DEPENDS "HAVE_LIBZ" - # Specifically throw a fatal error if a user tries to enable the zlib compressor without - # actually having the library available (as opposed to silently defaulting to OFF). - DEPENDS_ERROR ON "Failed to find zlib library" -) - -config_bool( - ENABLE_ZSTD - "Build the libzstd compressor extension" - DEFAULT OFF - DEPENDS "HAVE_LIBZSTD" - # Specifically throw a fatal error if a user tries to enable the zstd compressor without - # actually having the library available (as opposed to silently defaulting to OFF). - DEPENDS_ERROR ON "Failed to find zstd library" -) - -config_bool( - ENABLE_SODIUM - "Build the libsodium encryption extension" - DEFAULT OFF - DEPENDS "HAVE_LIBSODIUM" - # Specifically throw a fatal error if a user tries to enable the libsodium encryptor without - # actually having the library available (as opposed to silently defaulting to OFF). - DEPENDS_ERROR ON "Failed to find sodium library" -) - -config_bool( - ENABLE_TCMALLOC - "Use TCMalloc as the backend allocator" - DEFAULT OFF - DEPENDS "HAVE_LIBTCMALLOC" - # Specifically throw a fatal error if a user tries to enable the tcmalloc allocator without - # actually having the library available (as opposed to silently defaulting to OFF). - DEPENDS_ERROR ON "Failed to find tcmalloc library" -) - -set(default_optimize_level) -if("${WT_OS}" STREQUAL "windows") - set(default_optimize_level "/O2") -else() - set(default_optimize_level "-O3") -endif() -config_string( - CC_OPTIMIZE_LEVEL - "CC optimization level" - DEFAULT "${default_optimize_level}" -) - -config_string( - VERSION_MAJOR - "Major version number for WiredTiger" - DEFAULT 10 -) - -config_string( - VERSION_MINOR - "Minor version number for WiredTiger" - DEFAULT 0 -) - -config_string( - VERSION_PATCH - "Path version number for WiredTiger" - DEFAULT 1 -) - - -string(TIMESTAMP config_date "%Y-%m-%d") -config_string( - VERSION_STRING - "Version string for WiredTiger" - DEFAULT "\"WiredTiger ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH} (${config_date})\"" -) - -if(HAVE_DIAGNOSTIC AND (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")) - # Avoid setting diagnostic flags if we are building with Debug mode. - # CMakes Debug config sets compilation with debug symbols by default. - if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") - # Produce full symbolic debugging information. - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Z7") - # Ensure a PDB file can be generated for debugging symbols. - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG") - else() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g") - endif() -endif() - -if(WT_WIN) - # Check if we a using the dynamic or static run-time library. - if(DYNAMIC_CRT) - # Use the multithread-specific and DLL-specific version of the run-time library (MSVCRT.lib). - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /MD") - else() - # Use the multithread, static version of the run-time library. - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /MT") - endif() -endif() - -if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Release") - # Don't use the optimization level if we have specified a release config. - # CMakes Release config sets compilation to the highest optimization level - # by default. - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CC_OPTIMIZE_LEVEL}") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/configs/compile_test/pthread_cond_monotonic_test.c b/src/third_party/wiredtiger/build_cmake/configs/compile_test/pthread_cond_monotonic_test.c deleted file mode 100644 index 64645ec889f..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/compile_test/pthread_cond_monotonic_test.c +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2014-present MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include -#include -#include -#include - -int -main() -{ - int ret; - pthread_condattr_t condattr; - pthread_cond_t cond; - pthread_mutex_t mtx; - struct timespec ts; - - if ((ret = pthread_condattr_init(&condattr)) != 0) - exit(1); - if ((ret = pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) != 0) - exit(1); - if ((ret = pthread_cond_init(&cond, &condattr)) != 0) - exit(1); - if ((ret = pthread_mutex_init(&mtx, NULL)) != 0) - exit(1); - if ((ret = clock_gettime(CLOCK_MONOTONIC, &ts)) != 0) - exit(1); - ts.tv_sec += 1; - if ((ret = pthread_mutex_lock(&mtx)) != 0) - exit(1); - if ((ret = pthread_cond_timedwait(&cond, &mtx, &ts)) != 0 && ret != EINTR && ret != ETIMEDOUT) - exit(1); - - exit(0); -} diff --git a/src/third_party/wiredtiger/build_cmake/configs/modes.cmake b/src/third_party/wiredtiger/build_cmake/configs/modes.cmake deleted file mode 100644 index 74bf5cf8188..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/modes.cmake +++ /dev/null @@ -1,112 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -# Establishes build configuration modes we can use when compiling. - -include(CheckCCompilerFlag) - -set(build_modes None Debug Release) -if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") - set(no_omit_frame_flag "/Oy-") -else() - set(no_omit_frame_flag "-fno-omit-frame-pointer") -endif() - -# Create an ASAN build variant -if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") - set(asan_link_flags "/fsanitize=address") - set(asan_compiler_flag "/fsanitize=address") -elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") - set(asan_link_flags "-fsanitize=address -static-libsan") - set(asan_compiler_flag "-fsanitize=address") -else() - set(asan_link_flags "-fsanitize=address -static-libasan") - set(asan_compiler_flag "-fsanitize=address") -endif() - -# Needs to validate linker flags for the test to also pass. -set(CMAKE_REQUIRED_FLAGS "${asan_link_flags}") -# Check if the ASAN compiler flag is available. -check_c_compiler_flag("${asan_compiler_flag}" HAVE_ADDRESS_SANITIZER) -unset(CMAKE_REQUIRED_FLAGS) - -if(HAVE_ADDRESS_SANITIZER) - set(CMAKE_C_FLAGS_ASAN - "${CMAKE_C_FLAGS_DEBUG} ${asan_compiler_flag} ${no_omit_frame_flag}" CACHE STRING - "Flags used by the C compiler for ASan build type or configuration." FORCE) - - set(CMAKE_CXX_FLAGS_ASAN - "${CMAKE_CXX_FLAGS_DEBUG} ${asan_compiler_flag} ${no_omit_frame_flag}" CACHE STRING - "Flags used by the C++ compiler for ASan build type or configuration." FORCE) - - set(CMAKE_EXE_LINKER_FLAGS_ASAN - "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} ${asan_link_flags}" CACHE STRING - "Linker flags to be used to create executables for ASan build type." FORCE) - - set(CMAKE_SHARED_LINKER_FLAGS_ASAN - "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} ${asan_link_flags}" CACHE STRING - "Linker lags to be used to create shared libraries for ASan build type." FORCE) - - mark_as_advanced( - CMAKE_CXX_FLAGS_ASAN - CMAKE_C_FLAGS_ASAN - CMAKE_EXE_LINKER_FLAGS_ASAN - CMAKE_SHARED_LINKER_FLAGS_ASAN - ) - list(APPEND build_modes "ASan") -endif() - -# Create an UBSAN build variant -if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") - set(ubsan_link_flags "/fsanitize=undefined") - set(ubsan_compiler_flag "/fsanitize=undefined") -elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") - set(ubsan_link_flags "-fsanitize=undefined") - set(ubsan_compiler_flag "-fsanitize=undefined") -else() - set(ubsan_link_flags "-fsanitize=undefined -lubsan") - set(ubsan_compiler_flag "-fsanitize=undefined") -endif() - -# Needs to validate linker flags for the test to also pass. -set(CMAKE_REQUIRED_FLAGS "${ubsan_link_flags}") -# Check if the UBSAN compiler flag is available. -check_c_compiler_flag("${ubsan_compiler_flag}" HAVE_UB_SANITIZER) -unset(CMAKE_REQUIRED_FLAGS) - -if(HAVE_UB_SANITIZER) - set(CMAKE_C_FLAGS_UBSAN - "${CMAKE_C_FLAGS_DEBUG} ${ubsan_compiler_flag} ${no_omit_frame_flag}" CACHE STRING - "Flags used by the C compiler for UBSan build type or configuration." FORCE) - - set(CMAKE_CXX_FLAGS_UBSAN - "${CMAKE_CXX_FLAGS_DEBUG} ${ubsan_compiler_flag} ${no_omit_frame_flag}" CACHE STRING - "Flags used by the C++ compiler for UBSan build type or configuration." FORCE) - - set(CMAKE_EXE_LINKER_FLAGS_UBSAN - "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} ${ubsan_link_flags}" CACHE STRING - "Linker flags to be used to create executables for UBSan build type." FORCE) - - set(CMAKE_SHARED_LINKER_FLAGS_UBSAN - "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} ${ubsan_link_flags}" CACHE STRING - "Linker lags to be used to create shared libraries for UBSan build type." FORCE) - - mark_as_advanced( - CMAKE_CXX_FLAGS_UBSAN - CMAKE_C_FLAGS_UBSAN - CMAKE_EXE_LINKER_FLAGS_UBSAN - CMAKE_SHARED_LINKER_FLAGS_UBSAN - ) - list(APPEND build_modes "UBSan") -endif() -if(NOT CMAKE_BUILD_TYPE) - string(REPLACE ";" " " build_modes_doc "${build_modes}") - set(CMAKE_BUILD_TYPE "None" CACHE STRING "Choose the type of build, options are: ${build_modes_doc}." FORCE) -endif() - -set(CMAKE_CONFIGURATION_TYPES ${build_modes}) diff --git a/src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake deleted file mode 100644 index 4d1821c9e18..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/ppc64le/linux/config.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -set(WT_ARCH "ppc64le" CACHE STRING "") -set(WT_OS "linux" CACHE STRING "") -set(WT_POSIX ON CACHE BOOL "") - -# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions -# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based -# builds. -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) - -# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. -set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") - -# Allow assembler to detect '.sx' file extensions. -list(APPEND CMAKE_ASM_SOURCE_FILE_EXTENSION "sx") - -# Our ASM-based checksum utility 'crc32.sx' triggers unused-macros diagnostic errors -# when compiling. To avoid editing the original source, override the usage '-Wunused-macros' -# for this specific file. -set_source_files_properties(src/checksum/power8/crc32.sx PROPERTIES COMPILE_FLAGS -Wno-unused-macros) diff --git a/src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake deleted file mode 100644 index 639da9f297e..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/s390x/linux/config.cmake +++ /dev/null @@ -1,22 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -set(WT_ARCH "s390x" CACHE STRING "") -set(WT_OS "linux" CACHE STRING "") -set(WT_POSIX ON CACHE BOOL "") - -# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions -# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based -# builds. -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) - -# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. -set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") - -# Allow assembler to detect '.sx' file extensions. -list(APPEND CMAKE_ASM_SOURCE_FILE_EXTENSION "sx") diff --git a/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in b/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in deleted file mode 100644 index 5cdf5d357b0..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/wiredtiger_config.h.in +++ /dev/null @@ -1,177 +0,0 @@ -/*- - * Copyright (c) 2014-present MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#ifndef __WIREDTIGER_CONFIG_H_ -#define __WIREDTIGER_CONFIG_H_ - -/* Define to 1 to pause for debugger attach on failure. */ -#cmakedefine HAVE_ATTACH 1 - -/* LZ4 support automatically loaded. */ -#cmakedefine HAVE_BUILTIN_EXTENSION_LZ4 1 - -/* Snappy support automatically loaded. */ -#cmakedefine HAVE_BUILTIN_EXTENSION_SNAPPY 1 - -/* ZLIB support automatically loaded. */ -#cmakedefine HAVE_BUILTIN_EXTENSION_ZLIB 1 - -/* ZSTD support automatically loaded. */ -#cmakedefine HAVE_BUILTIN_EXTENSION_ZSTD 1 - -/* libsodium support automatically loaded. */ -#cmakedefine HAVE_BUILTIN_EXTENSION_SODIUM 1 - -/* Define to 1 if you have the `clock_gettime' function. */ -#cmakedefine HAVE_CLOCK_GETTIME 1 - -/* Define to 1 for diagnostic tests. */ -#cmakedefine HAVE_DIAGNOSTIC 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_DLFCN_H 1 - -/* Define to 1 if you have the `fallocate' function. */ -#cmakedefine HAVE_FALLOCATE 1 - -/* Define to 1 if you have the `fdatasync' function. */ -#cmakedefine HAVE_FDATASYNC 1 - -/* Define to 1 if you have the `ftruncate' function. */ -#cmakedefine HAVE_FTRUNCATE 1 - -/* Define to 1 if you have the `gettimeofday' function. */ -#cmakedefine HAVE_GETTIMEOFDAY 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the `dl' library (-ldl). */ -#cmakedefine HAVE_LIBDL 1 - -/* Define to 1 if you have the `lz4' library (-llz4). */ -#cmakedefine HAVE_LIBLZ4 1 - -/* Define to 1 if you have the `pthread' library (-lpthread). */ -#cmakedefine HAVE_LIBPTHREAD 1 - -/* Define to 1 if you have the `rt' library (-lrt). */ -#cmakedefine HAVE_LIBRT 1 - -/* Define to 1 if you have the `snappy' library (-lsnappy). */ -#cmakedefine HAVE_LIBSNAPPY 1 - -/* Define to 1 if the user has explictly enabled TCMalloc builds. */ -#cmakedefine ENABLE_TCMALLOC 1 - -/* - * To remain compatible with autoconf & scons builds, we - * define HAVE_LIBTCMALLOC for configuring our sources to actually - * include the tcmalloc headers, as opposed to the sources - * using ENABLE_TCMALLOC. - */ -#if defined ENABLE_TCMALLOC - /* Define to 1 if you have the `tcmalloc' library (-ltcmalloc). */ - #cmakedefine HAVE_LIBTCMALLOC 1 -#endif - -/* Define to 1 if you have the `z' library (-lz). */ -#cmakedefine HAVE_LIBZ 1 - -/* Define to 1 if you have the `zstd' library (-lzstd). */ -#cmakedefine HAVE_LIBZSTD 1 - -/* Define to 1 if you have the `sodium' library (-lsodium). */ -#cmakedefine HAVE_LIBSODIUM 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_MEMORY_H 1 - -/* Define to 1 to disable any crc32 hardware support. */ -#cmakedefine HAVE_NO_CRC32_HARDWARE - -/* Define to 1 if you have the `posix_fadvise' function. */ -#cmakedefine HAVE_POSIX_FADVISE 1 - -/* Define to 1 if you have the `posix_fallocate' function. */ -#cmakedefine HAVE_POSIX_FALLOCATE 1 - -/* Define to 1 if you have the `posix_madvise' function. */ -#cmakedefine HAVE_POSIX_MADVISE 1 - -/* Define to 1 if `posix_memalign' works. */ -#cmakedefine HAVE_POSIX_MEMALIGN 1 - -/* Define to 1 if pthread condition variables support monotonic clocks. */ -#cmakedefine HAVE_PTHREAD_COND_MONOTONIC 1; - -/* Define to 1 if you have the `setrlimit' function. */ -#cmakedefine HAVE_SETRLIMIT 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STDLIB_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STRING_H 1 - -/* Define to 1 if you have the `strtouq' function. */ -#cmakedefine HAVE_STRTOUQ 1 - -/* Define to 1 if you have the `sync_file_range' function. */ -#cmakedefine HAVE_SYNC_FILE_RANGE 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the `timer_create' function. */ -#cmakedefine HAVE_TIMER_CREATE 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_UNISTD_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_X86INTRIN_H 1 - -/* Spinlock type from mutex.h. */ -#cmakedefine SPINLOCK_TYPE @SPINLOCK_TYPE_CONFIG_VAR@ - -/* Define to 1 if the target system is big endian */ -#cmakedefine WORDS_BIGENDIAN 1 - -/* Version number of package */ -#define VERSION "@VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_PATCH@" - -/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most - significant byte first (like Motorola and SPARC, unlike Intel). */ -#if defined AC_APPLE_UNIVERSAL_BUILD -# if defined __BIG_ENDIAN__ -# define WORDS_BIGENDIAN 1 -# endif -#else -# ifndef WORDS_BIGENDIAN -/* # undef WORDS_BIGENDIAN */ -# endif -#endif - -/* Default alignment of buffers used for I/O. */ -#define WT_BUFFER_ALIGNMENT_DEFAULT @WT_BUFFER_ALIGNMENT_DEFAULT@ - -/* Define to 1 to support standalone build. */ -#cmakedefine WT_STANDALONE_BUILD 1 - -#ifndef _DARWIN_USE_64_BIT_INODE -# define _DARWIN_USE_64_BIT_INODE 1 -#endif - -#endif diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/darwin/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/x86/darwin/config.cmake deleted file mode 100644 index 2d3f7ead67a..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/darwin/config.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -set(WT_ARCH "x86" CACHE STRING "") -set(WT_OS "darwin" CACHE STRING "") -set(WT_POSIX ON CACHE BOOL "") diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake deleted file mode 100644 index 4cd2d6882e4..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/linux/config.cmake +++ /dev/null @@ -1,19 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -set(WT_ARCH "x86" CACHE STRING "") -set(WT_OS "linux" CACHE STRING "") -set(WT_POSIX ON CACHE BOOL "") - -# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions -# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based -# builds. -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE") - -# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. -set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/netbsd/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/x86/netbsd/config.cmake deleted file mode 100644 index f307d3b0110..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/netbsd/config.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -set(WT_ARCH "x86" CACHE STRING "") -set(WT_OS "netbsd" CACHE STRING "") -set(WT_POSIX ON CACHE BOOL "") - -# NetBSD requires buffers aligned to 512-byte (DEV_BSIZE) boundaries for O_DIRECT to work. -set(WT_BUFFER_ALIGNMENT_DEFAULT "512" CACHE STRING "") diff --git a/src/third_party/wiredtiger/build_cmake/configs/x86/windows/config.cmake b/src/third_party/wiredtiger/build_cmake/configs/x86/windows/config.cmake deleted file mode 100644 index 842708a1f9d..00000000000 --- a/src/third_party/wiredtiger/build_cmake/configs/x86/windows/config.cmake +++ /dev/null @@ -1,40 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -set(WT_ARCH "x86" CACHE STRING "") -set(WT_OS "windows" CACHE STRING "") -set(WT_POSIX OFF CACHE BOOL "") -set(SPINLOCK_TYPE "msvc" CACHE STRING "" FORCE) -# We force a static compilation to generate a ".lib" file. We can then -# additionally generate a dll file using a *DEF file. -set(ENABLE_STATIC ON CACHE BOOL "" FORCE) - -# Compile as C code . -add_compile_options(/TC) -# Inline expansion. -add_compile_options(/Ob1) -# Enable string pooling. -add_compile_options(/GF) -# Extern "C" does not throw. -add_compile_options(/EHsc) -# Separate functions for linker. -add_compile_options(/Gy) -# Conformance: wchar_t is a native type, not a typedef. -add_compile_options(/Zc:wchar_t) -# Use the __cdecl calling convention for all functions. -add_compile_options(/Gd) - -# Disable incremental linking. -string(APPEND win_link_flags " /INCREMENTAL:NO") -# Remove dead code. -string(APPEND win_link_flags " /OPT:REF") -# Allow executables to be randomly rebased at load time (enables virtual address allocation randomization). -string(APPEND win_link_flags " /DYNAMICBASE") -# Executable is compatible with the Windows Data Execution Prevention. -string(APPEND win_link_flags " /NXCOMPAT") -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${win_link_flags}") diff --git a/src/third_party/wiredtiger/build_cmake/helpers.cmake b/src/third_party/wiredtiger/build_cmake/helpers.cmake deleted file mode 100644 index 4dc449c0748..00000000000 --- a/src/third_party/wiredtiger/build_cmake/helpers.cmake +++ /dev/null @@ -1,652 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -include(CheckIncludeFiles) -include(CheckSymbolExists) -include(CheckLibraryExists) -include(CheckTypeSize) - -# Helper function for evaluating a list of dependencies. Mostly used by the -# "config_X" helpers to evaluate the dependencies required to enable the config -# option. -# depends - a list (semicolon seperated) of dependencies to evaluate. -# enabled - name of the output variable set with either 'ON' or 'OFF' (based -# on evaluated dependencies). Output variable is set in the callers scope. -function(eval_dependency depends enabled) - # If no dependencies were given then we default to an enabled state. - if(("${depends}" STREQUAL "") OR ("${depends}" STREQUAL "NOTFOUND")) - set(enabled ON PARENT_SCOPE) - return() - endif() - # Evaluate each dependency. - set(is_enabled ON) - foreach(dependency ${depends}) - string(REGEX REPLACE " +" ";" dependency "${dependency}") - if(NOT (${dependency})) - set(is_enabled OFF) - break() - endif() - endforeach() - set(enabled ${is_enabled} PARENT_SCOPE) -endfunction() - -# config_string(config_name description DEFAULT [DEPENDS ] [INTERNAL]) -# Defines a string configuration option. The configuration option is stored in the cmake cache -# and can be exported to the wiredtiger config header. -# config_name - name of the configuration option. -# description - docstring to describe the configuration option (viewable in the cmake-gui). -# DEFAULT - Default value of the configuration string. Used when not manually set -# by a cmake script or in the cmake-gui. -# DEPENDS - list of dependencies (semicolon seperated) required for the configuration string -# to be present and set in the cache. If any of the dependencies aren't met, the -# configuration value won't be present in the cache. -# INTERNAL - hides the configuration option from the cmake-gui by default. Useful if you don't want -# to expose the variable by default to the user i.e keep it internal to the implementation -# (but still need it in the cache). -function(config_string config_name description) - cmake_parse_arguments( - PARSE_ARGV - 2 - "CONFIG_STR" - "INTERNAL" - "DEFAULT;DEPENDS" - "" - ) - - if (NOT "${CONFIG_STR_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to config_str: ${CONFIG_STR_UNPARSED_ARGUMENTS}") - endif() - # We require a default value (not optional). - if ("${CONFIG_STR_DEFAULT}" STREQUAL "") - message(FATAL_ERROR "No default value passed") - endif() - - # Check that the configs dependencies are enabled before setting it to a visible enabled state. - eval_dependency("${CONFIG_STR_DEPENDS}" enabled) - set(default_value "${CONFIG_STR_DEFAULT}") - if(enabled) - # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state - # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. - if(${config_name}_DISABLED) - unset(${config_name}_DISABLED CACHE) - set(${config_name} ${default_value} CACHE STRING "${description}" FORCE) - else() - set(${config_name} ${default_value} CACHE STRING "${description}") - endif() - if (CONFIG_STR_INTERNAL) - # Mark as an advanced variable, hiding it from initial UI's views. - mark_as_advanced(FORCE ${config_name}) - endif() - else() - # Config doesn't meet dependency requirements, remove it from the cache and flag it as disabled. - unset(${config_name} CACHE) - set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) - endif() -endfunction() - -# config_choice(config_name description OPTIONS ) -# Defines a configuration option, bounded with pre-set toggleable values. The configuration option is stored -# in the cmake cache and can be exported to the wiredtiger config header. We default to the first *available* option in the -# list if the config has not been manually set by a cmake script or in the cmake-gui. -# config_name - name of the configuration option. -# description - docstring to describe the configuration option (viewable in the cmake-gui). -# OPTIONS - a list of option values that the configuration option can be set to. Each option is itself a semicolon -# seperated list consisting of ";;". -# * option-name: name of the given option stored in the ${config_name} cache variable and presented -# to users in the gui (usually something understandable). -# * config-name: an additional cached configuration variable that is made available if the option is selected. -# It is only present if the option is chosen, otherwise it is unset. -# * option-dependencies: dependencies required for the option to be made available. If its dependencies aren't met -# the given option will become un-selectable. -function(config_choice config_name description) - cmake_parse_arguments( - PARSE_ARGV - 2 - "CONFIG_OPT" - "" - "" - "OPTIONS" - ) - - if (NOT "${CONFIG_OPT_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to config_opt: ${CONFIG_OPT_UNPARSED_ARGUMENTS}") - endif() - # We require option values (not optional) - if ("${CONFIG_OPT_OPTIONS}" STREQUAL "") - message(FATAL_ERROR "No options passed") - endif() - - set(found_option ON) - set(found_pre_set OFF) - set(default_config_field "") - set(default_config_var "") - foreach(curr_option ${CONFIG_OPT_OPTIONS}) - list(LENGTH curr_option opt_length) - if (NOT opt_length EQUAL 3) - message(FATAL_ERROR "Invalid option format: ${curr_option}") - endif() - # We expect three items defined for each option. - list(GET curr_option 0 option_config_field) - list(GET curr_option 1 option_config_var) - list(GET curr_option 2 option_depends) - # Check that the options dependencies are enabled before setting it to a selectable state. - eval_dependency("${option_depends}" enabled) - if(enabled) - list(APPEND all_option_config_fields ${option_config_field}) - # The first valid/selectable option found will be the default config value. - if (found_option) - set(found_option OFF) - set(default_config_field "${option_config_field}") - set(default_config_var "${option_config_var}") - endif() - # Check if the option is already set with this given field. We don't want to override the configs value - # with a default value if its already been pre-set in the cache e.g. by early config scripts. - if("${${config_name}}" STREQUAL "${option_config_field}") - set(${option_config_var} ON CACHE INTERNAL "" FORCE) - set(${config_name}_CONFIG_VAR ${option_config_var} CACHE INTERNAL "" FORCE) - set(found_pre_set ON) - set(found_option OFF) - set(default_config_field "${option_config_field}") - set(default_config_var "${option_config_var}") - else() - # Clear the cache of the current set value. - set(${option_config_var} OFF CACHE INTERNAL "" FORCE) - endif() - else() - unset(${option_config_var} CACHE) - # Check if the option is already set with this given field - we want to clear it if so. - if ("${${config_name}_CONFIG_VAR}" STREQUAL "${option_config_var}") - unset(${config_name}_CONFIG_VAR CACHE) - endif() - if("${${config_name}}" STREQUAL "${option_config_field}") - unset(${config_name} CACHE) - endif() - endif() - endforeach() - # If the config hasn't been set we can load it with the default option found earlier. - if(NOT found_pre_set) - set(${default_config_var} ON CACHE INTERNAL "" FORCE) - set(${config_name} ${default_config_field} CACHE STRING ${description}) - set(${config_name}_CONFIG_VAR ${default_config_var} CACHE INTERNAL "" FORCE) - endif() - set_property(CACHE ${config_name} PROPERTY STRINGS ${all_option_config_fields}) -endfunction() - -# config_bool(config_name description DEFAULT [DEPENDS ] [DEPENDS_ERROR ]) -# Defines a boolean (ON/OFF) configuration option . The configuration option is stored in the cmake cache -# and can be exported to the wiredtiger config header. -# config_name - name of the configuration option. -# description - docstring to describe the configuration option (viewable in the cmake-gui). -# DEFAULT - default value of the configuration bool (ON/OFF). Used when not manually set -# by a cmake script or in the cmake-gui or when dependencies aren't met. -# DEPENDS - list of dependencies (semicolon seperated) required for the configuration bool -# to be set to the desired value. If any of the dependencies aren't met the configuration value -# will be set to its default value. -# DEPENDS_ERROR - specifically throw a fatal error when the configuration option is set to -# despite failing on its dependencies. This is mainly used for commandline-like options where you want -# to signal a specific error to the caller when dependencies aren't met e.g. toolchain is missing library (as opposed to -# silently defaulting). -function(config_bool config_name description) - cmake_parse_arguments( - PARSE_ARGV - 2 - "CONFIG_BOOL" - "" - "DEFAULT;DEPENDS" - "DEPENDS_ERROR" - ) - - if(NOT "${CONFIG_BOOL_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to config_bool: ${CONFIG_BOOL_UNPARSED_ARGUMENTS}") - endif() - # We require a default value (not optional). - if("${CONFIG_BOOL_DEFAULT}" STREQUAL "") - message(FATAL_ERROR "No default value passed") - endif() - - set(depends_err_value) - set(depends_err_message "") - # If DEPENDS_ERROR is specifically set, parse the value we want to throw an error on if the dependency fails. - if(CONFIG_BOOL_DEPENDS_ERROR) - list(LENGTH CONFIG_BOOL_DEPENDS_ERROR depends_error_length) - if(NOT depends_error_length EQUAL 2) - message(FATAL_ERROR "Invalid usage of DEPENDS_ERROR: requires ") - else() - list(GET CONFIG_BOOL_DEPENDS_ERROR 0 err_val) - if(err_val) - set(depends_err_value "1") - else() - set(depends_err_value "0") - endif() - list(GET CONFIG_BOOL_DEPENDS_ERROR 1 depends_err_message) - endif() - endif() - - # Check that the configs dependencies are enabled before setting it to a visible enabled state. - eval_dependency("${CONFIG_BOOL_DEPENDS}" enabled) - if(enabled) - # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. - # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. - if(${config_name}_DISABLED) - unset(${config_name}_DISABLED CACHE) - set(${config_name} ${CONFIG_BOOL_DEFAULT} CACHE STRING "${description}" FORCE) - else() - set(${config_name} ${CONFIG_BOOL_DEFAULT} CACHE STRING "${description}") - endif() - else() - set(config_value "0") - if (${${config_name}}) - set(config_value "1") - endif() - # If the user tries to set the config option to a given value when its dependencies - # are not met, throw an error (when DEPENDS_ERROR is explicitly set). - if(CONFIG_BOOL_DEPENDS_ERROR) - if(${depends_err_value} EQUAL ${config_value}) - message(FATAL_ERROR "Unable to set ${config_name}: ${depends_err_message}") - endif() - endif() - # Config doesn't meet dependency requirements, set its default state and flag it as disabled. - set(${config_name} OFF CACHE STRING "${description}" FORCE) - set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) - endif() -endfunction() - -# config_func(config_name description FUNC FILE [DEPENDS ] [LIBS ]) -# Defines a boolean (0/1) configuration option based on whether a given function symbol exists. -# The configuration option is stored in the cmake cache and can be exported to the wiredtiger config header. -# config_name - name of the configuration option. -# description - docstring to describe the configuration option (viewable in the cmake-gui). -# FUNC - function symbol we want to search for. -# FILE - header we expect the function symbol to be defined e.g a std header. -# DEPENDS - list of dependencies (semicolon seperated) required for the configuration to be evaluated. -# If any of the dependencies aren't met the configuration value will be set to '0' (false). -# LIBS - a list of any additional library dependencies needed to successfully link with the function symbol. -function(config_func config_name description) - cmake_parse_arguments( - PARSE_ARGV - 2 - "CONFIG_FUNC" - "" - "FUNC;DEPENDS;FILES;LIBS" - "" - ) - - if (NOT "${CONFIG_FUNC_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to config_func: ${CONFIG_FUNC_UNPARSED_ARGUMENTS}") - endif() - # We require an include header (not optional). - if ("${CONFIG_FUNC_FILES}" STREQUAL "") - message(FATAL_ERROR "No file list passed") - endif() - # We require a function symbol (not optional). - if ("${CONFIG_FUNC_FUNC}" STREQUAL "") - message(FATAL_ERROR "No function passed") - endif() - - # Check that the configs dependencies are enabled before setting it to a visible enabled state. - eval_dependency("${CONFIG_FUNC_DEPENDS}" enabled) - if(enabled) - set(CMAKE_REQUIRED_LIBRARIES "${CONFIG_FUNC_LIBS}") - if((NOT "${WT_ARCH}" STREQUAL "") AND (NOT "${WT_ARCH}" STREQUAL "")) - # 'check_symbol_exists' won't use our current cache when test compiling the function symbol. - # To get around this we need to ensure we manually forward WT_ARCH and WT_OS as a minimum. This is particularly - # needed if 'check_symbol_exists' will leverage one of our toolchain files. - set(CMAKE_REQUIRED_FLAGS "-DWT_ARCH=${WT_ARCH} -DWT_OS=${WT_OS}") - endif() - check_symbol_exists(${CONFIG_FUNC_FUNC} "${CONFIG_FUNC_FILES}" has_symbol_${config_name}) - set(CMAKE_REQUIRED_LIBRARIES) - set(CMAKE_REQUIRED_FLAGS) - set(has_symbol "0") - if(has_symbol_${config_name}) - set(has_symbol ${has_symbol_${config_name}}) - endif() - # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. - # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. - if(${config_name}_DISABLED) - unset(${config_name}_DISABLED CACHE) - set(${config_name} ${has_symbol} CACHE STRING "${description}" FORCE) - else() - set(${config_name} ${has_symbol} CACHE STRING "${description}") - endif() - # 'check_symbol_exists' sets our given temp variable into the cache. Clear this so it doesn't persist between - # configuration runs. - unset(has_symbol_${config_name} CACHE) - else() - # Config doesn't meet dependency requirements, set a disabled state. - set(${config_name} 0 CACHE INTERNAL "" FORCE) - set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) - endif() -endfunction() - - -# config_include(config_name description FILE [DEPENDS ]) -# Defines a boolean (0/1) configuration option based on whether a given include header exists. -# The configuration option is stored in the cmake cache and can be exported to the wiredtiger config header. -# config_name - name of the configuration option. -# description - docstring to describe the configuration option (viewable in the cmake-gui). -# FILE - header we want to search for e.g a std header. -# DEPENDS - list of dependencies (semicolon seperated) required for the configuration to be evaluated. -# If any of the dependencies aren't met the configuration value will be set to '0' (false). -function(config_include config_name description) - cmake_parse_arguments( - PARSE_ARGV - 2 - "CONFIG_INCLUDE" - "" - "FILE;DEPENDS" - "" - ) - - if (NOT "${CONFIG_INCLUDE_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to config_include: ${CONFIG_INCLUDE_UNPARSED_ARGUMENTS}") - endif() - # We require a include header (not optional). - if ("${CONFIG_INCLUDE_FILE}" STREQUAL "") - message(FATAL_ERROR "No include file passed") - endif() - - # Check that the configs dependencies are enabled before setting it to a visible enabled state. - eval_dependency("${CONFIG_INCLUDE_DEPENDS}" enabled) - if(enabled) - # 'check_include_files' won't use our current cache when test compiling the include header. - # To get around this we need to ensure we manually forward WT_ARCH and WT_OS as a minimum. This is particularly - # needed if 'check_include_files' will leverage one of our toolchain files. - if((NOT "${WT_ARCH}" STREQUAL "") AND (NOT "${WT_ARCH}" STREQUAL "")) - set(CMAKE_REQUIRED_FLAGS "-DWT_ARCH=${WT_ARCH} -DWT_OS=${WT_OS}") - endif() - check_include_files(${CONFIG_INCLUDE_FILE} has_include_${config_name}) - set(CMAKE_REQUIRED_FLAGS) - set(has_include "0") - if(has_include_${config_name}) - set(has_include ${has_include_${config_name}}) - endif() - # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. - # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. - if(${config_name}_DISABLED) - unset(${config_name}_DISABLED CACHE) - set(${config_name} ${has_include} CACHE STRING "${description}" FORCE) - else() - set(${config_name} ${has_include} CACHE STRING "${description}") - endif() - # 'check_include_files' sets our given temp variable into the cache. Clear this so it doesn't persist between - # configuration runs. - unset(has_include_${config_name} CACHE) - else() - set(${config_name} OFF CACHE INTERNAL "" FORCE) - set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) - endif() - # Set an internal cahce variable with the CPP include statement. We can use this when building out our config header. - if (${${config_name}}) - set(${config_name}_DECL "#include <${CONFIG_INCLUDE_FILE}>" CACHE INTERNAL "") - endif() -endfunction() - -# config_lib(config_name description LIB FUNC [DEPENDS ] [HEADER ]) -# Defines a boolean (0/1) configuration option based on whether a given library exists. -# The configuration option is stored in the cmake cache and can be exported to the wiredtiger config header. -# config_name - name of the configuration option. -# description - docstring to describe the configuration option (viewable in the cmake-gui). -# LIB - library we are searching for (defined as if we are linking against it e.g -lpthread). -# FUNC - function symbol we expect to be available to link against within the library. -# DEPENDS - list of dependencies (semicolon seperated) required for the configuration to be evaluated. -# If any of the dependencies aren't met the configuration value will be set to '0' (false). -function(config_lib config_name description) - cmake_parse_arguments( - PARSE_ARGV - 2 - "CONFIG_LIB" - "" - "LIB;DEPENDS;HEADER" - "" - ) - - if (NOT "${CONFIG_LIB_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to config_lib: ${CONFIG_LIB_UNPARSED_ARGUMENTS}") - endif() - # We require a library (not optional). - if ("${CONFIG_LIB_LIB}" STREQUAL "") - message(FATAL_ERROR "No library passed") - endif() - - # Check that the configs dependencies are enabled before setting it to a visible enabled state. - eval_dependency("${CONFIG_LIB_DEPENDS}" enabled) - if(enabled) - message("-- Looking for library ${CONFIG_LIB_LIB}") - # 'check_library_exists' won't use our current cache when test compiling the library. - # To get around this we need to ensure we manually forward WT_ARCH and WT_OS as a minimum. This is particularly - # needed if 'check_library_exists' will leverage one of our toolchain files. - if((NOT "${WT_ARCH}" STREQUAL "") AND (NOT "${WT_ARCH}" STREQUAL "")) - set(CMAKE_REQUIRED_FLAGS "-DWT_ARCH=${WT_ARCH} -DWT_OS=${WT_OS}") - endif() - find_library(has_lib_${config_name} ${CONFIG_LIB_LIB}) - set(CMAKE_REQUIRED_FLAGS) - set(has_lib "0") - if(has_lib_${config_name}) - set(has_lib ${has_lib_${config_name}}) - if (CONFIG_LIB_HEADER) - find_path(include_path_${config_name} ${CONFIG_LIB_HEADER}) - if (include_path_${config_name}) - message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}, include path ${include_path_${config_name}}") - include_directories(${include_path_${config_name}}) - else() - message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}") - endif() - unset(include_path_${config_name} CACHE) - else() - message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}") - endif() - else() - message("-- Looking for library ${CONFIG_LIB_LIB}: NOT found") - endif() - # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. - # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. - if(${config_name}_DISABLED) - unset(${config_name}_DISABLED CACHE) - set(${config_name} ${has_lib} CACHE STRING "${description}" FORCE) - else() - set(${config_name} ${has_lib} CACHE STRING "${description}") - endif() - # 'check_library_exists' sets our given temp variable into the cache. Clear this so it doesn't persist between - # configuration runs. - unset(has_lib_${config_name} CACHE) - else() - message("-- Not looking for library ${CONFIG_LIB_LIB}: disabled") - set(${config_name} 0 CACHE INTERNAL "" FORCE) - set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) - endif() -endfunction() - -# config_compile(config_name description SOURCE [DEPENDS ] [LIBS ]) -# Defines a boolean (0/1) configuration option based on whether a source file can be successfully compiled and run. Used -# to determine if more fine grained functionality is supported on a given target environment (beyond what function -# symbols, libraries and headers are available). The configuration option is stored in the cmake cache and can be -# exported to the wiredtiger config header. -# config_name - name of the configuration option. -# description - docstring to describe the configuration option (viewable in the cmake-gui). -# SOURCE - specific source file we want to test compile. -# DEPENDS - list of dependencies (semicolon seperated) required for the configuration to be evaluated. -# If any of the dependencies aren't met the configuration value will be set to '0' (false). -# LIBS - a list of any additional library dependencies needed to successfully compile the source. -function(config_compile config_name description) - cmake_parse_arguments( - PARSE_ARGV - 2 - "CONFIG_COMPILE" - "" - "SOURCE;DEPENDS;LIBS" - "" - ) - - if (NOT "${CONFIG_COMPILE_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to config_compile: ${CONFIG_COMPILE_UNPARSED_ARGUMENTS}") - endif() - # We require a source file (not optional). - if ("${CONFIG_COMPILE_SOURCE}" STREQUAL "") - message(FATAL_ERROR "No source passed") - endif() - - # Check that the configs dependencies are enabled before setting it to a visible enabled state. - eval_dependency("${CONFIG_COMPILE_DEPENDS}" enabled) - if(enabled) - # Test compile the source file. - try_run( - can_run_${config_name} can_compile_${config_name} - ${CMAKE_CURRENT_BINARY_DIR} - ${CONFIG_COMPILE_SOURCE} - CMAKE_FLAGS "-DWT_ARCH=${WT_ARCH}" "-DWT_OS=${WT_OS}" - LINK_LIBRARIES "${CONFIG_COMPILE_LIBS}" - ) - set(can_run "0") - if((NOT "${can_run_${config_name}}" STREQUAL "FAILED_TO_RUN") AND - ("${can_run_${config_name}}" STREQUAL "0")) - set(can_run "1") - endif() - # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. - # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. - if(${config_name}_DISABLED) - unset(${config_name}_DISABLED CACHE) - set(${config_name} ${can_run} CACHE STRING "${description}" FORCE) - else() - set(${config_name} ${can_run} CACHE STRING "${description}") - endif() - # 'try_run' sets our given temp variable into the cache. Clear this so it doesn't persist between - # configuration runs. - unset(can_run_${config_name} CACHE) - unset(can_compile_${config_name} CACHE) - else() - set(${config_name} 0 CACHE INTERNAL "" FORCE) - set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) - endif() -endfunction() - -# test_type_size(type output_size) -# Helper function that tests for a given types size and returns its value if found. -# type - name of the type to test. -# output_size - name of the output variable, set with either the types size or "" (empty string) -# if not found. -# EXTRA_INCLUDES - extra/optional include files to access the given type e.g. a custom typedef in an include header. -function(test_type_size type output_size) - cmake_parse_arguments( - PARSE_ARGV - 2 - "TEST_TYPE" - "" - "" - "EXTRA_INCLUDES" - ) - - if (NOT "${TEST_TYPE_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to assert_type: ${TEST_TYPE_UNPARSED_ARGUMENTS}") - endif() - - set(CMAKE_EXTRA_INCLUDE_FILES "${TEST_TYPE_EXTRA_INCLUDES}") - check_type_size(${type} TEST_TYPE) - set(CMAKE_EXTRA_INCLUDE_FILES) - - if(NOT HAVE_TEST_TYPE) - set(${output_size} "" PARENT_SCOPE) - else() - set(${output_size} ${TEST_TYPE} PARENT_SCOPE) - endif() -endfunction() - -# assert_type_size(type size) -# Wrapper function around 'test_type_size' that additionally asserts whether the given types meets an expected size. -# Throws a fatal error if the type is not found or doesn't equal the expected size. -# type - eame of the type to test. -# size - expected size of the type. -# EXTRA_INCLUDES - extra/optional include files to access the given type e.g. a custom typedef in an include header. -function(assert_type_size type size) - cmake_parse_arguments( - PARSE_ARGV - 2 - "ASSERT_TYPE" - "" - "" - "EXTRA_INCLUDES" - ) - - if (NOT "${ASSERT_TYPE_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to assert_type: ${ASSERT_TYPE_UNPARSED_ARGUMENTS}") - endif() - - set(additional_args "") - if(${ASSERT_TYPE_EXTRA_INCLUDES}) - set(additional_args "EXTRA_INCLUDES ${ASSERT_TYPE_EXTRA_INCLUDES}") - endif() - test_type_size(${type} output_type_size ${additional_args}) - - if(${output_type_size} EQUAL "") - # Type does not exist. - message(FATAL_ERROR "Type assertion failed: ${type} does not exists") - endif() - - if((NOT ${size} EQUAL 0) AND (NOT ${output_type_size} EQUAL ${size})) - # Type does not meet size assertion. - message(FATAL_ERROR "Type assertion failed: ${type} does not equal size ${size}") - endif() -endfunction() - -# parse_filelist_source(filelist output_var) -# A helper function that parses the list of sources (usually found in "dist/filelist"). This returning a list of -# sources that can then be used to generate the necessary build rules for the wiredtiger library. Additionally -# uses the config values "WT_ARCH" and "WT_OS" when extracting platform specific sources from the file list. -# filelist - Destination of 'filelist' file. -# output_var - name of the output variable that will be set with the parsed sources. Output variable is set in -# the callers scope. -function(parse_filelist_source filelist output_var) - set(arch_host "") - set(plat_host "") - # Determine architecture host for our filelist parse. - if(WT_X86) - set(arch_host "X86_HOST") - elseif(WT_AARCH64) - set(arch_host "ARM64_HOST") - elseif(WT_PPC64) - set(arch_host "POWERPC_HOST") - elseif(WT_S390X) - set(arch_host "ZSERIES_HOST") - endif() - # Determine platform host for our filelist parse. - if(WT_POSIX) - set(plat_host "POSIX_HOST") - elseif(WT_WIN) - set(plat_host "WINDOWS_HOST") - endif() - - # Read file list and parse into list. - file(READ "${filelist}" contents NEWLINE_CONSUME) - string(REGEX REPLACE "\n" ";" contents "${contents}") - set(output_files "") - foreach(file ${contents}) - if(${file} MATCHES "^#.*$") - continue() - endif() - string(REGEX REPLACE "[ \t\r]+" ";" file_contents ${file}) - list(LENGTH file_contents file_contents_len) - if (file_contents_len EQUAL 1) - list(APPEND output_files ${file}) - elseif(file_contents_len EQUAL 2) - list(GET file_contents 0 file_name) - list(GET file_contents 1 file_group) - if ((${file_group} STREQUAL "${plat_host}") OR (${file_group} STREQUAL "${arch_host}")) - list(APPEND output_files ${file_name}) - get_filename_component(file_ext ${file_name} EXT) - # POWERPC and ZSERIES hosts use the '.sx' extension for their ASM files. We need to - # manually tell CMake to ASM compile these files otherwise it will ignore them during - # compilation process. - if("${file_ext}" STREQUAL ".sx") - set_source_files_properties(${file_name} PROPERTIES LANGUAGE ASM) - endif() - endif() - else() - message(FATAL_ERROR "filelist (${filelist}) has an unexpected format [Invalid Line: \"${file}]\"") - endif() - endforeach() - set(${output_var} ${output_files} PARENT_SCOPE) -endfunction() diff --git a/src/third_party/wiredtiger/build_cmake/install/install.cmake b/src/third_party/wiredtiger/build_cmake/install/install.cmake deleted file mode 100644 index ec9b30234d6..00000000000 --- a/src/third_party/wiredtiger/build_cmake/install/install.cmake +++ /dev/null @@ -1,37 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information -# - -include(GNUInstallDirs) - -# Library installs - -# Define the public headers for wiredtiger library to be used when installing the target. -set_property( - TARGET wiredtiger - PROPERTY PUBLIC_HEADER - ${CMAKE_BINARY_DIR}/include/wiredtiger.h - ${CMAKE_SOURCE_DIR}/src/include/wiredtiger_ext.h -) -# Set the version property of the wiredtiger library so we can export a versioned install. -set_target_properties(wiredtiger PROPERTIES VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") - -# Install the wiredtiger library target. -install(TARGETS wiredtiger - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -) - -# Create our wiredtiger pkgconfig (for POSIX builds). -if(WT_POSIX) - configure_file(${CMAKE_CURRENT_LIST_DIR}/wiredtiger.pc.in wiredtiger.pc @ONLY) - install( - FILES ${CMAKE_BINARY_DIR}/wiredtiger.pc - DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig - ) -endif() diff --git a/src/third_party/wiredtiger/build_cmake/install/wiredtiger.pc.in b/src/third_party/wiredtiger/build_cmake/install/wiredtiger.pc.in deleted file mode 100644 index 9ed407b7fe4..00000000000 --- a/src/third_party/wiredtiger/build_cmake/install/wiredtiger.pc.in +++ /dev/null @@ -1,11 +0,0 @@ -prefix=@CMAKE_INSTALL_PREFIX@ -exec_prefix=@CMAKE_INSTALL_PREFIX@ -libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ -includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ - -Name: WiredTiger -Description: The WiredTiger Data Engine -Requires: -Version: @VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_PATCH@ -Libs: -L${libdir} -lwiredtiger -Cflags: -I${includedir} diff --git a/src/third_party/wiredtiger/build_cmake/strict/cl_strict.cmake b/src/third_party/wiredtiger/build_cmake/strict/cl_strict.cmake deleted file mode 100644 index 625bf9e88d5..00000000000 --- a/src/third_party/wiredtiger/build_cmake/strict/cl_strict.cmake +++ /dev/null @@ -1,21 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -# Warning level 3. -list(APPEND win_c_flags "/WX") -# Ignore warning about mismatched const qualifiers. -list(APPEND win_c_flags "/wd4090") -# Ignore deprecated functions. -list(APPEND win_c_flags "/wd4996") -# Complain about unreferenced format parameter. -list(APPEND win_c_flags "/we4100") -# Enable security check. -list(APPEND win_c_flags "/GS") - -# Set our base compiler flags that can be used by the rest of our build. -set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${win_c_flags}" CACHE INTERNAL "" FORCE) diff --git a/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake b/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake deleted file mode 100644 index f87c988fe02..00000000000 --- a/src/third_party/wiredtiger/build_cmake/strict/clang_strict.cmake +++ /dev/null @@ -1,65 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -list(APPEND clang_base_c_flags "-Weverything") -list(APPEND clang_base_c_flags "-Werror") -list(APPEND clang_base_c_flags "-Wno-cast-align") -list(APPEND clang_base_c_flags "-Wno-documentation-unknown-command") -list(APPEND clang_base_c_flags "-Wno-format-nonliteral") -list(APPEND clang_base_c_flags "-Wno-packed") -list(APPEND clang_base_c_flags "-Wno-padded") -list(APPEND clang_base_c_flags "-Wno-reserved-id-macro") -list(APPEND clang_base_c_flags "-Wno-zero-length-array") - -# We should turn on cast-qual, but not as a fatal error: see WT-2690. -# For now, turn it off. -list(APPEND clang_base_c_flags "-Wno-cast-qual") - -# Turn off clang thread-safety-analysis, it doesn't like some of the -# code patterns in WiredTiger. -list(APPEND clang_base_c_flags "-Wno-thread-safety-analysis") - -# On Centos 7.3.1611, system header files aren't compatible with -# -Wdisabled-macro-expansion. -list(APPEND clang_base_c_flags "-Wno-disabled-macro-expansion") - -# We occasionally use an extra semicolon to indicate an empty loop or -# conditional body. -list(APPEND clang_base_c_flags "-Wno-extra-semi-stmt") - -# Ignore unrecognized options. -list(APPEND clang_base_c_flags "-Wno-unknown-warning-option") - -if(WT_DARWIN AND (CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.1)) - # Apple clang has its own numbering system, and older OS X - # releases need some special love. Turn off some flags for - # Apple's clang 4.1: - # Apple clang version 4.1 - # ((tags/Apple/clang-421.11.66) (based on LLVM 3.1svn) - list(APPEND clang_base_c_flags "-Wno-attributes") - list(APPEND clang_base_c_flags "-Wno-pedantic") - list(APPEND clang_base_c_flags "-Wno-unused-command-line-argument") -endif() - -if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 10) - # Clang 10+ has added additional on-by-default diagnostics that isn't - # compatible with some of the code patterns in WiredTiger. - list(APPEND clang_base_c_flags "-Wno-implicit-fallthrough") - list(APPEND clang_base_c_flags "-Wno-implicit-int-float-conversion") -endif() - -if(WT_DARWIN AND NOT CMAKE_CROSSCOMPILING) - # If we are not cross-compiling, we can safely disable this diagnostic. - # Its incompatible with strict diagnostics when including external - # libraries that are not in the default linker path - # e.g. linking zlib/snappy/... from /usr/local/. - list(APPEND clang_base_c_flags "-Wno-poison-system-directories") -endif() - -# Set our base clang flags to ensure it propogates to the rest of our build. -set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${clang_base_c_flags}" CACHE INTERNAL "" FORCE) diff --git a/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake b/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake deleted file mode 100644 index 420a4457676..00000000000 --- a/src/third_party/wiredtiger/build_cmake/strict/gcc_strict.cmake +++ /dev/null @@ -1,79 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -list(APPEND gcc_base_c_flags "-Wall") -list(APPEND gcc_base_c_flags "-Wextra") -list(APPEND gcc_base_c_flags "-Werror") -list(APPEND gcc_base_c_flags "-Waggregate-return") -list(APPEND gcc_base_c_flags "-Wbad-function-cast") -list(APPEND gcc_base_c_flags "-Wcast-align") -list(APPEND gcc_base_c_flags "-Wdeclaration-after-statement") -list(APPEND gcc_base_c_flags "-Wdouble-promotion") -list(APPEND gcc_base_c_flags "-Wfloat-equal") -list(APPEND gcc_base_c_flags "-Wformat-nonliteral") -list(APPEND gcc_base_c_flags "-Wformat-security") -list(APPEND gcc_base_c_flags "-Wformat=2") -list(APPEND gcc_base_c_flags "-Winit-self") -list(APPEND gcc_base_c_flags "-Wjump-misses-init") -list(APPEND gcc_base_c_flags "-Wmissing-declarations") -list(APPEND gcc_base_c_flags "-Wmissing-field-initializers") -list(APPEND gcc_base_c_flags "-Wmissing-prototypes") -list(APPEND gcc_base_c_flags "-Wnested-externs") -list(APPEND gcc_base_c_flags "-Wold-style-definition") -list(APPEND gcc_base_c_flags "-Wpacked") -list(APPEND gcc_base_c_flags "-Wpointer-arith") -list(APPEND gcc_base_c_flags "-Wpointer-sign") -list(APPEND gcc_base_c_flags "-Wredundant-decls") -list(APPEND gcc_base_c_flags "-Wshadow") -list(APPEND gcc_base_c_flags "-Wsign-conversion") -list(APPEND gcc_base_c_flags "-Wstrict-prototypes") -list(APPEND gcc_base_c_flags "-Wswitch-enum") -list(APPEND gcc_base_c_flags "-Wundef") -list(APPEND gcc_base_c_flags "-Wuninitialized") -list(APPEND gcc_base_c_flags "-Wunreachable-code") -list(APPEND gcc_base_c_flags "-Wunused") -list(APPEND gcc_base_c_flags "-Wwrite-strings") - -# Non-fatal informational warnings. -# We don't turn on the unsafe-loop-optimizations warning after gcc7, -# it's too noisy to tolerate. Regardless, don't fail even when it's -# configured. -list(APPEND gcc_base_c_flags "-Wno-error=unsafe-loop-optimizations") -if(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 4.7) - list(APPEND gcc_base_c_flags "-Wno-c11-extensions") - list(APPEND gcc_base_c_flags "-Wunsafe-loop-optimizations") -elseif(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 5) - list(APPEND gcc_base_c_flags "-Wunsafe-loop-optimizations") -endif() - -if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 5) - list(APPEND gcc_base_c_flags "-Wformat-signedness") - list(APPEND gcc_base_c_flags "-Wjump-misses-init") - list(APPEND gcc_base_c_flags "-Wredundant-decls") - list(APPEND gcc_base_c_flags "-Wunused-macros") - list(APPEND gcc_base_c_flags "-Wvariadic-macros") -endif() -if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 6) - list(APPEND gcc_base_c_flags "-Wduplicated-cond") - list(APPEND gcc_base_c_flags "-Wlogical-op") - list(APPEND gcc_base_c_flags "-Wunused-const-variable=2") -endif() -if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 7) - list(APPEND gcc_base_c_flags "-Walloca") - list(APPEND gcc_base_c_flags "-Walloc-zero") - list(APPEND gcc_base_c_flags "-Wduplicated-branches") - list(APPEND gcc_base_c_flags "-Wformat-overflow=2") - list(APPEND gcc_base_c_flags "-Wformat-truncation=2") - list(APPEND gcc_base_c_flags "-Wrestrict") -endif() -if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 8) - list(APPEND gcc_base_c_flags "-Wmultistatement-macros") -endif() - -# Set our base gcc flags to ensure it propogates to the rest of our build. -set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${gcc_base_c_flags}" CACHE INTERNAL "" FORCE) diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake deleted file mode 100644 index 98cc2a8892b..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_clang.cmake +++ /dev/null @@ -1,15 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(TRIPLE_TARGET "aarch64-unknown-linux-gnu") - set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) - set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake deleted file mode 100644 index b82bbfe9b94..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/aarch64/linux/plat_gcc.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(CROSS_COMPILER_PREFIX "aarch64-linux-gnu-") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/cl.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/cl.cmake deleted file mode 100644 index d6467238ce3..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/cl.cmake +++ /dev/null @@ -1,13 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -cmake_minimum_required(VERSION 3.10.0) - -set(CMAKE_C_COMPILER "cl.exe") -set(CMAKE_CXX_COMPILER "cl.exe") -set(CMAKE_ASM_COMPILER "cl.exe") diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake deleted file mode 100644 index 5b789f1134f..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/clang.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -cmake_minimum_required(VERSION 3.10.0) - -set(CMAKE_C_COMPILER "clang") -set(CMAKE_C_COMPILER_ID "Clang") - -set(CMAKE_CXX_COMPILER "clang++") -set(CMAKE_CXX_COMPILER_ID "Clang++") - -set(CMAKE_ASM_COMPILER "clang") -set(CMAKE_ASM_COMPILER_ID "Clang") - -if(NOT "${COMPILE_DEFINITIONS}" STREQUAL "") - ### Additional check to overcome check_[symbol|include|function]_exits using toolchain file without passing WT_ARCH and WT_OS. - string(REGEX MATCH "-DWT_ARCH=([A-Za-z0-9]+) -DWT_OS=([A-Za-z0-9]+)" _ ${COMPILE_DEFINITIONS}) - set(wt_config_arch ${CMAKE_MATCH_1}) - set(wt_config_os ${CMAKE_MATCH_2}) -else() - set(wt_config_arch ${WT_ARCH}) - set(wt_config_os ${WT_OS}) -endif() - -# Include any platform specific clang configurations and flags e.g. target-tuple, flags. -if((NOT "${wt_config_arch}" STREQUAL "") AND (NOT "${wt_config_os}" STREQUAL "")) - if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/${wt_config_arch}/${wt_config_os}/plat_clang.cmake") - message(FATAL_ERROR "(${wt_config_arch}/${wt_config_os}) directory does not have a plat_clang.cmake file") - endif() - include("${CMAKE_CURRENT_LIST_DIR}/${wt_config_arch}/${wt_config_os}/plat_clang.cmake") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake deleted file mode 100644 index 0fa64ed0037..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/gcc.cmake +++ /dev/null @@ -1,31 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -cmake_minimum_required(VERSION 3.10.0) - -if(NOT "${COMPILE_DEFINITIONS}" STREQUAL "") - ### Additional check to overcome check_[symbol|include|function]_exits using toolchain file without passing WT_ARCH and WT_OS. - string(REGEX MATCH "-DWT_ARCH=([A-Za-z0-9]+) -DWT_OS=([A-Za-z0-9]+)" _ ${COMPILE_DEFINITIONS}) - set(wt_config_arch ${CMAKE_MATCH_1}) - set(wt_config_os ${CMAKE_MATCH_2}) -else() - set(wt_config_arch ${WT_ARCH}) - set(wt_config_os ${WT_OS}) -endif() - -# Include any platform specific gcc configurations and flags e.g. target-tuple, flags. -if((NOT "${wt_config_arch}" STREQUAL "") AND (NOT "${wt_config_os}" STREQUAL "")) - if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/${wt_config_arch}/${wt_config_os}/plat_gcc.cmake") - message(FATAL_ERROR "(${wt_config_arch}/${wt_config_os}) directory does not have a plat_gcc.cmake file") - endif() - include("${CMAKE_CURRENT_LIST_DIR}/${wt_config_arch}/${wt_config_os}/plat_gcc.cmake") -endif() - -set(CMAKE_C_COMPILER "${CROSS_COMPILER_PREFIX}gcc") -set(CMAKE_CXX_COMPILER "${CROSS_COMPILER_PREFIX}g++") -set(CMAKE_ASM_COMPILER "${CROSS_COMPILER_PREFIX}gcc") diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_clang.cmake deleted file mode 100644 index 2353ba1b7f7..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_clang.cmake +++ /dev/null @@ -1,17 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -cmake_minimum_required(VERSION 3.10.0) - -if(NOT TOOLCHAIN_ROOT) - set(TOOLCHAIN_ROOT "/opt/mongodbtoolchain/v3") -endif() - -set(CMAKE_C_COMPILER "${TOOLCHAIN_ROOT}/bin/clang") -set(CMAKE_CXX_COMPILER "${TOOLCHAIN_ROOT}/bin/clang++") -set(CMAKE_ASM_COMPILER "${TOOLCHAIN_ROOT}/bin/clang") diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake deleted file mode 100644 index 136d4a33f4f..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake +++ /dev/null @@ -1,17 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -cmake_minimum_required(VERSION 3.10.0) - -if(NOT TOOLCHAIN_ROOT) - set(TOOLCHAIN_ROOT "/opt/mongodbtoolchain/v3") -endif() - -set(CMAKE_C_COMPILER "${TOOLCHAIN_ROOT}/bin/gcc") -set(CMAKE_CXX_COMPILER "${TOOLCHAIN_ROOT}/bin/g++") -set(CMAKE_ASM_COMPILER "${TOOLCHAIN_ROOT}/bin/gcc") diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake deleted file mode 100644 index f03d4fdbd97..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_clang.cmake +++ /dev/null @@ -1,15 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(TRIPLE_TARGET "powerpc64le-unknown-linux-gnu") - set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) - set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake deleted file mode 100644 index 47e706c0d39..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/ppc64le/linux/plat_gcc.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(CROSS_COMPILER_PEFIX "powerpc64le-linux-gnu-") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake deleted file mode 100644 index f643b9c8df8..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_clang.cmake +++ /dev/null @@ -1,15 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(TRIPLE_TARGET "s390x-ibm-linux-gnu") - set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) - set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake deleted file mode 100644 index abfff342dfb..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/s390x/linux/plat_gcc.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(CROSS_COMPILER_PEFIX "s390x-linux-gnu-") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake deleted file mode 100644 index e7130be256f..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_clang.cmake +++ /dev/null @@ -1,15 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(TRIPLE_TARGET "x86_64-apple-darwin") - set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) - set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake deleted file mode 100644 index 7a463e46d96..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/darwin/plat_gcc.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -# We are not cross-compiling if our system is Darwin, hence the "x86_64-apple-darwin-" -# prefix is not necessary when we are not cross-compiling. Just default to the host -# installed 'gcc' binary. -if(CMAKE_CROSSCOMPILING) - set(CROSS_COMPILER_PREFIX "x86_64-apple-darwin-") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake deleted file mode 100644 index 126a3a1bbbe..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_clang.cmake +++ /dev/null @@ -1,15 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(TRIPLE_TARGET "x86_64-pc-linux-gnu") - set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) - set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") - set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") -endif() diff --git a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake b/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake deleted file mode 100644 index 2b36fb667d5..00000000000 --- a/src/third_party/wiredtiger/build_cmake/toolchains/x86/linux/plat_gcc.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# All rights reserved. -# -# See the file LICENSE for redistribution information. -# - -if(CMAKE_CROSSCOMPILING) - set(CROSS_COMPILER_PREFIX "x86_64-linux-gnu-") -endif() diff --git a/src/third_party/wiredtiger/cmake/README.md b/src/third_party/wiredtiger/cmake/README.md new file mode 100644 index 00000000000..75fa960da6a --- /dev/null +++ b/src/third_party/wiredtiger/cmake/README.md @@ -0,0 +1,157 @@ +# Building WiredTiger with CMake + +### Build Dependencies + +To build with CMake we **require** the following dependencies: + +* `cmake` : Official CMake install instructions found here: https://cmake.org/install/ + * *WiredTiger supports CMake 3.10+* +* `ninja` : Official ninja install instructions found here: https://ninja-build.org/ +* `python3-dev` : Consult your package management application for installation instructions. This is needed for building with Python support. +* `swig`: Consult your package management application for installation instructions. This is needed for building with Python support. + +We also strongly suggest the following dependencies are also installed (for improved build times): + +* `ccache` : Official ccache download instructions found here: https://ccache.dev/download.html + +##### Package Manager Instructions + +Alternatively you can use your system's package manager to install the dependencies listed above. Depending on the system, the following commands can be run: + +###### Install commands for Ubuntu & Debian (tested on Ubuntu 18.04) + +```bash +sudo apt-get install cmake cmake-curses-gui +sudo apt-get install ccache +sudo apt-get install ninja-build +sudo apt-get install python3-dev swig +``` + +###### Install commands for Mac (using HomeBrew) + +```bash +brew install ninja +brew install ccache +brew install cmake +brew install python +brew install swig +``` + +###### Install commands for Windows (using Chocolatey) + +```bash +choco install cmake +choco install ninja +choco install ccache --version=3.7.9 +choco install swig +choco install python --pre +``` + +### Building the WiredTiger Library + +> *The below commands are written for Linux and Darwin hosts. Windows instructions coming soon!* + +Building the WiredTiger library is relatively straightforward. Navigate to the top level of the WiredTiger repository and run the following commands: + +###### Configure your build + +```bash +# Create a new directory to run your build from +$ mkdir build && cd build +# Run the cmake configure step. Note: '-G Ninja' tells CMake to generate a ninja build +$ cmake -G Ninja ../. +... +-- Configuring done +-- Generating done +-- Build files have been written to: /home/wiredtiger/build +``` + +*See [Configuration Options](#configuration-options) for additional configuration options.* + +###### Run your build + +In the same directory you configured your build, run the `ninja` command to start the build: + +```bash +$ ninja +... +[211/211 (100%) 2.464s] Creating library symlink libwiredtiger.so +``` + +*Note: Ninja doesn't need a `-j` option; it knows how many cores are available.* + +###### Configuration Options + +There are a number of additional configuration options you can pass to the CMake configuration step. A summary of some important options you will come to know: + +* `-DENABLE_STATIC=1` : Compile WiredTiger as a static library +* `-DENABLE_LZ4=1` : Build the lz4 compressor extension +* `-DENABLE_SNAPPY=1` : Build the snappy compressor extension +* `-DENABLE_ZLIB=1` : Build the zlib compressor extension +* `-DENABLE_ZSTD=1` : Build the libzstd compressor extension +* `-DENABLE_SODIUM=1` : Build the libsodium encryptor extension +* `-DHAVE_DIAGNOSTIC=1` : Enable WiredTiger diagnostics +* `-DHAVE_ATTACH=1` : Enable to pause for debugger attach on failure +* `-DENABLE_STRICT=1` : Compile with strict compiler warnings enabled +* `-DENABLE_PYTHON=1` : Compile the python API +* `-DCMAKE_INSTALL_PREFIX=` : Path to install directory + +--- + +An example of using the above configuration options during the configuration step: + +```bash +$ cmake -DENABLE_STATIC=1 -DENABLE_LZ4=1 -DENABLE_SNAPPY=1 -DENABLE_ZLIB=1 -DENABLE_ZSTD=1 -DHAVE_DIAGNOSTIC=1 -DHAVE_ATTACH=1 -DENABLE_STRICT=1 -G Ninja ../. +``` + +--- + +You can further look at all the available configuration options (and also dynamically change them!) by running `ccmake` in your build directory: + +```bash +$ cd build +$ ccmake . +``` + +*The configuration options can also be viewed in `cmake/configs/base.cmake`*. + +###### Switching between GCC and Clang (POSIX only) + +By default CMake will use your default system compiler (`cc`). If you want to use a specific toolchain you can pass a toolchain file! We have provided a toolchain file for both GCC (`cmake/toolchains/gcc.cmake`) and Clang (`cmake/toolchains/clang.cmake`). To use either toolchain you can pass the `-DCMAKE_TOOLCHAIN_FILE=` to the CMake configuration step. For example: + +*Using the GCC Toolchain* + +```bash +$ cd build +$ cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/gcc.cmake -G Ninja ../. +``` + +*Using the Clang Toolchain* + +```bash +$ cd build +$ cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -G Ninja ../. +``` + +### Running WiredTiger C Tests + +The WiredTiger CMake build makes available a suite of C/C++ based tests (separate from the Python testsuite). The run all the tests available, ensure you're in the build directory and execute: + +```bash +ctest -j$(nproc) +``` + +To run with verbose output: + +```bash +ctest -j$(nproc) -VV +``` + +To run a specfic test: + +```bash +# Note: -R specifies a regex, where any matching test will be run +ctest -R -j$(nproc) +``` + +See `ctest --help` for a range of additional supported options. diff --git a/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake new file mode 100644 index 00000000000..94447ac9ca8 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake @@ -0,0 +1,34 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# +include(CheckCCompilerFlag) + +set(WT_ARCH "aarch64" CACHE STRING "") +set(WT_OS "linux" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") + +# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions +# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based +# builds. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) + +# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. +set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") + +# ARMv8-A is the 64-bit ARM architecture, turn on the optional CRC instructions. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc" CACHE STRING "" FORCE) + +check_c_compiler_flag("-moutline-atomics" has_moutline_atomics) + +# moutline-atomics preserves backwards compatibility with Arm v8.0 systems but also supports +# using Arm v8.1 atomics. The latter can massively improve performance on larger Arm systems. +# The flag was back ported to gcc8, 9 and is the default in gcc10+. See if the compiler supports +# the flag. +if(has_moutline_atomics) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -moutline-atomics" CACHE STRING "" FORCE) +endif() +unset(has_moutline_atomics CACHE) diff --git a/src/third_party/wiredtiger/cmake/configs/auto.cmake b/src/third_party/wiredtiger/cmake/configs/auto.cmake new file mode 100644 index 00000000000..3691da331a8 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/auto.cmake @@ -0,0 +1,325 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +include(cmake/helpers.cmake) + +### Auto configure options and checks that we can infer from our toolchain environment. + +## Assert type sizes. +assert_type_size("size_t" 8) +assert_type_size("ssize_t" 8) +assert_type_size("time_t" 8) +assert_type_size("off_t" 0) +assert_type_size("uintptr_t" 0) +test_type_size("uintmax_t" u_intmax_size) +test_type_size("unsigned long long" u_long_long_size) +set(default_uintmax_def " ") +if(${u_intmax_size} STREQUAL "") + if(${unsigned long long} STREQUAL "") + set(default_uintmax_def "typedef unsigned long uintmax_t\\;") + else() + set(default_uintmax_def "typedef unsigned long long uintmax_t\\;") + endif() +endif() + +set(default_offt_def) +if("${WT_OS}" STREQUAL "windows") + set(default_offt_def "typedef int64_t wt_off_t\\;") +else() + set(default_offt_def "typedef off_t wt_off_t\\;") +endif() + +config_string( + off_t_decl + "off_t type declaration." + DEFAULT "${default_offt_def}" + INTERNAL +) + +config_string( + uintprt_t_decl + "uintptr_t type declaration." + DEFAULT "${default_uintmax_def}" + INTERNAL +) + +config_include( + HAVE_SYS_TYPES_H + "Include header sys/types.h exists." + FILE "sys/types.h" +) + +config_include( + HAVE_INTTYPES_H + "Include header inttypes.h exists." + FILE "inttypes.h" +) + +config_include( + HAVE_STDARG_H + "Include header stdarg.h exists." + FILE "stdarg.h" +) + +config_include( + HAVE_STDBOOL_H + "Include header stdbool.h exists." + FILE "stdbool.h" +) + +config_include( + HAVE_STDINT_H + "Include header stdint.h exists." + FILE "stdint.h" +) + +config_include( + HAVE_STDLIB_H + "Include header stdlib.h exists." + FILE "stdlib.h" +) + +config_include( + HAVE_STDIO_H + "Include header stdio.h exists." + FILE "stdio.h" +) + +config_include( + HAVE_STRINGS_H + "Include header strings.h exists." + FILE "strings.h" +) + +config_include( + HAVE_STRING_H + "Include header string.h exists." + FILE "string.h" +) + +config_include( + HAVE_SYS_STAT_H + "Include header sys/stat.h exists." + FILE "sys/stat.h" +) + +config_include( + HAVE_UNISTD_H + "Include header unistd.h exists." + FILE "unistd.h" +) + +config_include( + HAVE_X86INTRIN_H + "Include header x86intrin.h exists." + FILE "x86intrin.h" +) + +config_include( + HAVE_DLFCN_H + "Include header dlfcn.h exists." + FILE "dlfcn.h" +) + +config_include( + HAVE_MEMORY_H + "Include header memory.h exists." + FILE "memory.h" +) + +config_func( + HAVE_CLOCK_GETTIME + "Function clock_gettime exists." + FUNC "clock_gettime" + FILES "time.h" +) + +config_func( + HAVE_FALLOCATE + "Function fallocate exists." + FUNC "fallocate" + FILES "fcntl.h" +) + +config_func( + HAVE_FDATASYNC + "Function fdatasync exists." + FUNC "fdatasync" + FILES "unistd.h" + DEPENDS "NOT WT_DARWIN" +) + +config_func( + HAVE_FTRUNCATE + "Function ftruncate exists." + FUNC "ftruncate" + FILES "unistd.h;sys/types.h" +) + +config_func( + HAVE_GETTIMEOFDAY + "Function gettimeofday exists." + FUNC "gettimeofday" + FILES "sys/time.h" +) + +config_func( + HAVE_POSIX_FADVISE + "Function posix_fadvise exists." + FUNC "posix_fadvise" + FILES "fcntl.h" +) + +config_func( + HAVE_POSIX_FALLOCATE + "Function posix_fallocate exists." + FUNC "posix_fallocate" + FILES "fcntl.h" +) + +config_func( + HAVE_POSIX_MADVISE + "Function posix_madvise exists." + FUNC "posix_madvise" + FILES "sys/mman.h" +) + +config_func( + HAVE_POSIX_MEMALIGN + "Function posix_memalign exists." + FUNC "posix_memalign" + FILES "stdlib.h" +) + +config_func( + HAVE_SETRLIMIT + "Function setrlimit exists." + FUNC "setrlimit" + FILES "sys/time.h;sys/resource.h" +) + +config_func( + HAVE_STRTOUQ + "Function strtouq exists." + FUNC "strtouq" + FILES "stdlib.h" +) + +config_func( + HAVE_SYNC_FILE_RANGE + "Function sync_file_range exists." + FUNC "sync_file_range" + FILES "fcntl.h" +) + +config_func( + HAVE_TIMER_CREATE + "Function timer_create exists." + FUNC "timer_create" + FILES "signal.h;time.h" + LIBS "rt" +) + +config_lib( + HAVE_LIBPTHREAD + "Pthread library exists." + LIB "pthread" +) + +config_lib( + HAVE_LIBRT + "rt library exists." + LIB "rt" +) + +config_lib( + HAVE_LIBDL + "dl library exists." + LIB "dl" +) + +config_lib( + HAVE_LIBLZ4 + "lz4 library exists." + LIB "lz4" + HEADER "lz4.h" +) + +config_lib( + HAVE_LIBSNAPPY + "snappy library exists." + LIB "snappy" + HEADER "snappy.h" +) + +config_lib( + HAVE_LIBZ + "zlib library exists." + LIB "z" + HEADER "zlib.h" +) + +config_lib( + HAVE_LIBZSTD + "zstd library exists." + LIB "zstd" + HEADER "zstd.h" +) + +config_lib( + HAVE_LIBSODIUM + "sodium library exists." + LIB "sodium" + HEADER "sodium.h" +) + +config_lib( + HAVE_LIBTCMALLOC + "tcmalloc library exists." + LIB "tcmalloc" +) + +config_compile( + HAVE_PTHREAD_COND_MONOTONIC + "If pthread condition variables support monotonic clocks." + SOURCE "${CMAKE_CURRENT_LIST_DIR}/compile_test/pthread_cond_monotonic_test.c" + LIBS "pthread" + DEPENDS "HAVE_LIBPTHREAD" +) + +include(TestBigEndian) +test_big_endian(is_big_endian) +if(NOT is_big_endian) + set(is_big_endian FALSE) +endif() +config_bool( + WORDS_BIGENDIAN + "If the target system is big endian" + DEFAULT ${is_big_endian} +) + +set(wiredtiger_includes_decl) +if(HAVE_SYS_TYPES_H) + list(APPEND wiredtiger_includes_decl "#include ") +endif() +if(HAVE_INTTYPES_H AND (NOT "${WT_OS}" STREQUAL "windows")) + list(APPEND wiredtiger_includes_decl "#include ") +endif() +if(HAVE_STDARG_H) + list(APPEND wiredtiger_includes_decl "#include ") +endif() +if(HAVE_STDBOOL_H) + list(APPEND wiredtiger_includes_decl "#include ") +endif() +if(HAVE_STDINT_H) + list(APPEND wiredtiger_includes_decl "#include ") +endif() +if(HAVE_STDIO_H) + list(APPEND wiredtiger_includes_decl "#include ") +endif() +string(REGEX REPLACE ";" "\n" wiredtiger_includes_decl "${wiredtiger_includes_decl}") diff --git a/src/third_party/wiredtiger/cmake/configs/base.cmake b/src/third_party/wiredtiger/cmake/configs/base.cmake new file mode 100644 index 00000000000..098859f2734 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/base.cmake @@ -0,0 +1,241 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +include(cmake/helpers.cmake) + +# WiredTiger-related configuration options. + +config_choice( + WT_ARCH + "Target architecture for WiredTiger" + OPTIONS + "x86;WT_X86;" + "aarch64;WT_AARCH64;" + "ppc64le;WT_PPC64;" + "s390x;WT_S390X;" +) + +config_choice( + WT_OS + "Target OS for WiredTiger" + OPTIONS + "darwin;WT_DARWIN;" + "windows;WT_WIN;" + "linux;WT_LINUX;" +) + +config_bool( + WT_POSIX + "Is a posix platform" + DEFAULT ON + DEPENDS "WT_LINUX OR WT_DARWIN" +) + +config_string( + WT_BUFFER_ALIGNMENT_DEFAULT + "WiredTiger buffer boundary aligment" + DEFAULT 0 +) + +config_bool( + HAVE_DIAGNOSTIC + "Enable WiredTiger diagnostics" + DEFAULT OFF +) + +config_bool( + HAVE_ATTACH + "Enable to pause for debugger attach on failure" + DEFAULT OFF +) + +config_bool( + ENABLE_STATIC + "Compile as a static library" + DEFAULT OFF +) + +config_bool( + ENABLE_STRICT + "Compile with strict compiler warnings enabled" + DEFAULT OFF +) + +config_bool( + ENABLE_PYTHON + "Configure the python API" + DEFAULT OFF +) + +config_string( + PYTHON3_REQUIRED_VERSION + "Exact Python version to use when building the Python API. \ + By default, when this configuration is unset, CMake will preference the \ + highest python version found to be installed in the users system path. \ + Expected format of version string: major[.minor[.patch]]" + DEFAULT "" + DEPENDS "ENABLE_PYTHON" +) + +config_bool( + WT_STANDALONE_BUILD + "Support standalone build" + DEFAULT ON +) + +config_bool( + HAVE_NO_CRC32_HARDWARE + "Disable any crc32 hardware support" + DEFAULT OFF +) + +config_bool( + DYNAMIC_CRT + "Link with the MSVCRT DLL version" + DEFAULT OFF + DEPENDS "WT_WIN" +) + +config_choice( + SPINLOCK_TYPE + "Set a spinlock type" + OPTIONS + "pthread;SPINLOCK_PTHREAD_MUTEX;HAVE_LIBPTHREAD" + "gcc;SPINLOCK_GCC;" + "msvc;SPINLOCK_MSVC;WT_WIN" + "pthread_adaptive;SPINLOCK_PTHREAD_ADAPTIVE;HAVE_LIBPTHREAD" +) + +config_bool( + ENABLE_LZ4 + "Build the lz4 compressor extension" + DEFAULT OFF + DEPENDS "HAVE_LIBLZ4" + # Specifically throw a fatal error if a user tries to enable the lz4 compressor without + # actually having the library available (as opposed to silently defaulting to OFF). + DEPENDS_ERROR ON "Failed to find lz4 library" +) + +config_bool( + ENABLE_SNAPPY + "Build the snappy compressor extension" + DEFAULT OFF + DEPENDS "HAVE_LIBSNAPPY" + # Specifically throw a fatal error if a user tries to enable the snappy compressor without + # actually having the library available (as opposed to silently defaulting to OFF). + DEPENDS_ERROR ON "Failed to find snappy library" +) + +config_bool( + ENABLE_ZLIB + "Build the zlib compressor extension" + DEFAULT OFF + DEPENDS "HAVE_LIBZ" + # Specifically throw a fatal error if a user tries to enable the zlib compressor without + # actually having the library available (as opposed to silently defaulting to OFF). + DEPENDS_ERROR ON "Failed to find zlib library" +) + +config_bool( + ENABLE_ZSTD + "Build the libzstd compressor extension" + DEFAULT OFF + DEPENDS "HAVE_LIBZSTD" + # Specifically throw a fatal error if a user tries to enable the zstd compressor without + # actually having the library available (as opposed to silently defaulting to OFF). + DEPENDS_ERROR ON "Failed to find zstd library" +) + +config_bool( + ENABLE_SODIUM + "Build the libsodium encryption extension" + DEFAULT OFF + DEPENDS "HAVE_LIBSODIUM" + # Specifically throw a fatal error if a user tries to enable the libsodium encryptor without + # actually having the library available (as opposed to silently defaulting to OFF). + DEPENDS_ERROR ON "Failed to find sodium library" +) + +config_bool( + ENABLE_TCMALLOC + "Use TCMalloc as the backend allocator" + DEFAULT OFF + DEPENDS "HAVE_LIBTCMALLOC" + # Specifically throw a fatal error if a user tries to enable the tcmalloc allocator without + # actually having the library available (as opposed to silently defaulting to OFF). + DEPENDS_ERROR ON "Failed to find tcmalloc library" +) + +set(default_optimize_level) +if("${WT_OS}" STREQUAL "windows") + set(default_optimize_level "/O2") +else() + set(default_optimize_level "-O3") +endif() +config_string( + CC_OPTIMIZE_LEVEL + "CC optimization level" + DEFAULT "${default_optimize_level}" +) + +config_string( + VERSION_MAJOR + "Major version number for WiredTiger" + DEFAULT 10 +) + +config_string( + VERSION_MINOR + "Minor version number for WiredTiger" + DEFAULT 0 +) + +config_string( + VERSION_PATCH + "Path version number for WiredTiger" + DEFAULT 1 +) + + +string(TIMESTAMP config_date "%Y-%m-%d") +config_string( + VERSION_STRING + "Version string for WiredTiger" + DEFAULT "\"WiredTiger ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH} (${config_date})\"" +) + +if(HAVE_DIAGNOSTIC AND (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")) + # Avoid setting diagnostic flags if we are building with Debug mode. + # CMakes Debug config sets compilation with debug symbols by default. + if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") + # Produce full symbolic debugging information. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Z7") + # Ensure a PDB file can be generated for debugging symbols. + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG") + else() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g") + endif() +endif() + +if(WT_WIN) + # Check if we a using the dynamic or static run-time library. + if(DYNAMIC_CRT) + # Use the multithread-specific and DLL-specific version of the run-time library (MSVCRT.lib). + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /MD") + else() + # Use the multithread, static version of the run-time library. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /MT") + endif() +endif() + +if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Release") + # Don't use the optimization level if we have specified a release config. + # CMakes Release config sets compilation to the highest optimization level + # by default. + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CC_OPTIMIZE_LEVEL}") +endif() diff --git a/src/third_party/wiredtiger/cmake/configs/compile_test/pthread_cond_monotonic_test.c b/src/third_party/wiredtiger/cmake/configs/compile_test/pthread_cond_monotonic_test.c new file mode 100644 index 00000000000..64645ec889f --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/compile_test/pthread_cond_monotonic_test.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2014-present MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include +#include +#include +#include + +int +main() +{ + int ret; + pthread_condattr_t condattr; + pthread_cond_t cond; + pthread_mutex_t mtx; + struct timespec ts; + + if ((ret = pthread_condattr_init(&condattr)) != 0) + exit(1); + if ((ret = pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) != 0) + exit(1); + if ((ret = pthread_cond_init(&cond, &condattr)) != 0) + exit(1); + if ((ret = pthread_mutex_init(&mtx, NULL)) != 0) + exit(1); + if ((ret = clock_gettime(CLOCK_MONOTONIC, &ts)) != 0) + exit(1); + ts.tv_sec += 1; + if ((ret = pthread_mutex_lock(&mtx)) != 0) + exit(1); + if ((ret = pthread_cond_timedwait(&cond, &mtx, &ts)) != 0 && ret != EINTR && ret != ETIMEDOUT) + exit(1); + + exit(0); +} diff --git a/src/third_party/wiredtiger/cmake/configs/modes.cmake b/src/third_party/wiredtiger/cmake/configs/modes.cmake new file mode 100644 index 00000000000..74bf5cf8188 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/modes.cmake @@ -0,0 +1,112 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +# Establishes build configuration modes we can use when compiling. + +include(CheckCCompilerFlag) + +set(build_modes None Debug Release) +if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") + set(no_omit_frame_flag "/Oy-") +else() + set(no_omit_frame_flag "-fno-omit-frame-pointer") +endif() + +# Create an ASAN build variant +if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") + set(asan_link_flags "/fsanitize=address") + set(asan_compiler_flag "/fsanitize=address") +elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") + set(asan_link_flags "-fsanitize=address -static-libsan") + set(asan_compiler_flag "-fsanitize=address") +else() + set(asan_link_flags "-fsanitize=address -static-libasan") + set(asan_compiler_flag "-fsanitize=address") +endif() + +# Needs to validate linker flags for the test to also pass. +set(CMAKE_REQUIRED_FLAGS "${asan_link_flags}") +# Check if the ASAN compiler flag is available. +check_c_compiler_flag("${asan_compiler_flag}" HAVE_ADDRESS_SANITIZER) +unset(CMAKE_REQUIRED_FLAGS) + +if(HAVE_ADDRESS_SANITIZER) + set(CMAKE_C_FLAGS_ASAN + "${CMAKE_C_FLAGS_DEBUG} ${asan_compiler_flag} ${no_omit_frame_flag}" CACHE STRING + "Flags used by the C compiler for ASan build type or configuration." FORCE) + + set(CMAKE_CXX_FLAGS_ASAN + "${CMAKE_CXX_FLAGS_DEBUG} ${asan_compiler_flag} ${no_omit_frame_flag}" CACHE STRING + "Flags used by the C++ compiler for ASan build type or configuration." FORCE) + + set(CMAKE_EXE_LINKER_FLAGS_ASAN + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} ${asan_link_flags}" CACHE STRING + "Linker flags to be used to create executables for ASan build type." FORCE) + + set(CMAKE_SHARED_LINKER_FLAGS_ASAN + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} ${asan_link_flags}" CACHE STRING + "Linker lags to be used to create shared libraries for ASan build type." FORCE) + + mark_as_advanced( + CMAKE_CXX_FLAGS_ASAN + CMAKE_C_FLAGS_ASAN + CMAKE_EXE_LINKER_FLAGS_ASAN + CMAKE_SHARED_LINKER_FLAGS_ASAN + ) + list(APPEND build_modes "ASan") +endif() + +# Create an UBSAN build variant +if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") + set(ubsan_link_flags "/fsanitize=undefined") + set(ubsan_compiler_flag "/fsanitize=undefined") +elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") + set(ubsan_link_flags "-fsanitize=undefined") + set(ubsan_compiler_flag "-fsanitize=undefined") +else() + set(ubsan_link_flags "-fsanitize=undefined -lubsan") + set(ubsan_compiler_flag "-fsanitize=undefined") +endif() + +# Needs to validate linker flags for the test to also pass. +set(CMAKE_REQUIRED_FLAGS "${ubsan_link_flags}") +# Check if the UBSAN compiler flag is available. +check_c_compiler_flag("${ubsan_compiler_flag}" HAVE_UB_SANITIZER) +unset(CMAKE_REQUIRED_FLAGS) + +if(HAVE_UB_SANITIZER) + set(CMAKE_C_FLAGS_UBSAN + "${CMAKE_C_FLAGS_DEBUG} ${ubsan_compiler_flag} ${no_omit_frame_flag}" CACHE STRING + "Flags used by the C compiler for UBSan build type or configuration." FORCE) + + set(CMAKE_CXX_FLAGS_UBSAN + "${CMAKE_CXX_FLAGS_DEBUG} ${ubsan_compiler_flag} ${no_omit_frame_flag}" CACHE STRING + "Flags used by the C++ compiler for UBSan build type or configuration." FORCE) + + set(CMAKE_EXE_LINKER_FLAGS_UBSAN + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} ${ubsan_link_flags}" CACHE STRING + "Linker flags to be used to create executables for UBSan build type." FORCE) + + set(CMAKE_SHARED_LINKER_FLAGS_UBSAN + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} ${ubsan_link_flags}" CACHE STRING + "Linker lags to be used to create shared libraries for UBSan build type." FORCE) + + mark_as_advanced( + CMAKE_CXX_FLAGS_UBSAN + CMAKE_C_FLAGS_UBSAN + CMAKE_EXE_LINKER_FLAGS_UBSAN + CMAKE_SHARED_LINKER_FLAGS_UBSAN + ) + list(APPEND build_modes "UBSan") +endif() +if(NOT CMAKE_BUILD_TYPE) + string(REPLACE ";" " " build_modes_doc "${build_modes}") + set(CMAKE_BUILD_TYPE "None" CACHE STRING "Choose the type of build, options are: ${build_modes_doc}." FORCE) +endif() + +set(CMAKE_CONFIGURATION_TYPES ${build_modes}) diff --git a/src/third_party/wiredtiger/cmake/configs/ppc64le/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/ppc64le/linux/config.cmake new file mode 100644 index 00000000000..4d1821c9e18 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/ppc64le/linux/config.cmake @@ -0,0 +1,27 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +set(WT_ARCH "ppc64le" CACHE STRING "") +set(WT_OS "linux" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") + +# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions +# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based +# builds. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) + +# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. +set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") + +# Allow assembler to detect '.sx' file extensions. +list(APPEND CMAKE_ASM_SOURCE_FILE_EXTENSION "sx") + +# Our ASM-based checksum utility 'crc32.sx' triggers unused-macros diagnostic errors +# when compiling. To avoid editing the original source, override the usage '-Wunused-macros' +# for this specific file. +set_source_files_properties(src/checksum/power8/crc32.sx PROPERTIES COMPILE_FLAGS -Wno-unused-macros) diff --git a/src/third_party/wiredtiger/cmake/configs/s390x/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/s390x/linux/config.cmake new file mode 100644 index 00000000000..639da9f297e --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/s390x/linux/config.cmake @@ -0,0 +1,22 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +set(WT_ARCH "s390x" CACHE STRING "") +set(WT_OS "linux" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") + +# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions +# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based +# builds. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE" CACHE STRING "" FORCE) + +# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. +set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") + +# Allow assembler to detect '.sx' file extensions. +list(APPEND CMAKE_ASM_SOURCE_FILE_EXTENSION "sx") diff --git a/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in b/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in new file mode 100644 index 00000000000..5cdf5d357b0 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in @@ -0,0 +1,177 @@ +/*- + * Copyright (c) 2014-present MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#ifndef __WIREDTIGER_CONFIG_H_ +#define __WIREDTIGER_CONFIG_H_ + +/* Define to 1 to pause for debugger attach on failure. */ +#cmakedefine HAVE_ATTACH 1 + +/* LZ4 support automatically loaded. */ +#cmakedefine HAVE_BUILTIN_EXTENSION_LZ4 1 + +/* Snappy support automatically loaded. */ +#cmakedefine HAVE_BUILTIN_EXTENSION_SNAPPY 1 + +/* ZLIB support automatically loaded. */ +#cmakedefine HAVE_BUILTIN_EXTENSION_ZLIB 1 + +/* ZSTD support automatically loaded. */ +#cmakedefine HAVE_BUILTIN_EXTENSION_ZSTD 1 + +/* libsodium support automatically loaded. */ +#cmakedefine HAVE_BUILTIN_EXTENSION_SODIUM 1 + +/* Define to 1 if you have the `clock_gettime' function. */ +#cmakedefine HAVE_CLOCK_GETTIME 1 + +/* Define to 1 for diagnostic tests. */ +#cmakedefine HAVE_DIAGNOSTIC 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_DLFCN_H 1 + +/* Define to 1 if you have the `fallocate' function. */ +#cmakedefine HAVE_FALLOCATE 1 + +/* Define to 1 if you have the `fdatasync' function. */ +#cmakedefine HAVE_FDATASYNC 1 + +/* Define to 1 if you have the `ftruncate' function. */ +#cmakedefine HAVE_FTRUNCATE 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#cmakedefine HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `dl' library (-ldl). */ +#cmakedefine HAVE_LIBDL 1 + +/* Define to 1 if you have the `lz4' library (-llz4). */ +#cmakedefine HAVE_LIBLZ4 1 + +/* Define to 1 if you have the `pthread' library (-lpthread). */ +#cmakedefine HAVE_LIBPTHREAD 1 + +/* Define to 1 if you have the `rt' library (-lrt). */ +#cmakedefine HAVE_LIBRT 1 + +/* Define to 1 if you have the `snappy' library (-lsnappy). */ +#cmakedefine HAVE_LIBSNAPPY 1 + +/* Define to 1 if the user has explictly enabled TCMalloc builds. */ +#cmakedefine ENABLE_TCMALLOC 1 + +/* + * To remain compatible with autoconf & scons builds, we + * define HAVE_LIBTCMALLOC for configuring our sources to actually + * include the tcmalloc headers, as opposed to the sources + * using ENABLE_TCMALLOC. + */ +#if defined ENABLE_TCMALLOC + /* Define to 1 if you have the `tcmalloc' library (-ltcmalloc). */ + #cmakedefine HAVE_LIBTCMALLOC 1 +#endif + +/* Define to 1 if you have the `z' library (-lz). */ +#cmakedefine HAVE_LIBZ 1 + +/* Define to 1 if you have the `zstd' library (-lzstd). */ +#cmakedefine HAVE_LIBZSTD 1 + +/* Define to 1 if you have the `sodium' library (-lsodium). */ +#cmakedefine HAVE_LIBSODIUM 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_MEMORY_H 1 + +/* Define to 1 to disable any crc32 hardware support. */ +#cmakedefine HAVE_NO_CRC32_HARDWARE + +/* Define to 1 if you have the `posix_fadvise' function. */ +#cmakedefine HAVE_POSIX_FADVISE 1 + +/* Define to 1 if you have the `posix_fallocate' function. */ +#cmakedefine HAVE_POSIX_FALLOCATE 1 + +/* Define to 1 if you have the `posix_madvise' function. */ +#cmakedefine HAVE_POSIX_MADVISE 1 + +/* Define to 1 if `posix_memalign' works. */ +#cmakedefine HAVE_POSIX_MEMALIGN 1 + +/* Define to 1 if pthread condition variables support monotonic clocks. */ +#cmakedefine HAVE_PTHREAD_COND_MONOTONIC 1; + +/* Define to 1 if you have the `setrlimit' function. */ +#cmakedefine HAVE_SETRLIMIT 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_STDLIB_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_STRING_H 1 + +/* Define to 1 if you have the `strtouq' function. */ +#cmakedefine HAVE_STRTOUQ 1 + +/* Define to 1 if you have the `sync_file_range' function. */ +#cmakedefine HAVE_SYNC_FILE_RANGE 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the `timer_create' function. */ +#cmakedefine HAVE_TIMER_CREATE 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_X86INTRIN_H 1 + +/* Spinlock type from mutex.h. */ +#cmakedefine SPINLOCK_TYPE @SPINLOCK_TYPE_CONFIG_VAR@ + +/* Define to 1 if the target system is big endian */ +#cmakedefine WORDS_BIGENDIAN 1 + +/* Version number of package */ +#define VERSION "@VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_PATCH@" + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Default alignment of buffers used for I/O. */ +#define WT_BUFFER_ALIGNMENT_DEFAULT @WT_BUFFER_ALIGNMENT_DEFAULT@ + +/* Define to 1 to support standalone build. */ +#cmakedefine WT_STANDALONE_BUILD 1 + +#ifndef _DARWIN_USE_64_BIT_INODE +# define _DARWIN_USE_64_BIT_INODE 1 +#endif + +#endif diff --git a/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake new file mode 100644 index 00000000000..2d3f7ead67a --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake @@ -0,0 +1,11 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +set(WT_ARCH "x86" CACHE STRING "") +set(WT_OS "darwin" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") diff --git a/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake new file mode 100644 index 00000000000..4cd2d6882e4 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake @@ -0,0 +1,19 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +set(WT_ARCH "x86" CACHE STRING "") +set(WT_OS "linux" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") + +# Linux requires '_GNU_SOURCE' to be defined for access to GNU/Linux extension functions +# e.g. Access to O_DIRECT on Linux. Append this macro to our compiler flags for Linux-based +# builds. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE") + +# Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. +set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "") diff --git a/src/third_party/wiredtiger/cmake/configs/x86/netbsd/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/netbsd/config.cmake new file mode 100644 index 00000000000..f307d3b0110 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/x86/netbsd/config.cmake @@ -0,0 +1,14 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +set(WT_ARCH "x86" CACHE STRING "") +set(WT_OS "netbsd" CACHE STRING "") +set(WT_POSIX ON CACHE BOOL "") + +# NetBSD requires buffers aligned to 512-byte (DEV_BSIZE) boundaries for O_DIRECT to work. +set(WT_BUFFER_ALIGNMENT_DEFAULT "512" CACHE STRING "") diff --git a/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake new file mode 100644 index 00000000000..842708a1f9d --- /dev/null +++ b/src/third_party/wiredtiger/cmake/configs/x86/windows/config.cmake @@ -0,0 +1,40 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +set(WT_ARCH "x86" CACHE STRING "") +set(WT_OS "windows" CACHE STRING "") +set(WT_POSIX OFF CACHE BOOL "") +set(SPINLOCK_TYPE "msvc" CACHE STRING "" FORCE) +# We force a static compilation to generate a ".lib" file. We can then +# additionally generate a dll file using a *DEF file. +set(ENABLE_STATIC ON CACHE BOOL "" FORCE) + +# Compile as C code . +add_compile_options(/TC) +# Inline expansion. +add_compile_options(/Ob1) +# Enable string pooling. +add_compile_options(/GF) +# Extern "C" does not throw. +add_compile_options(/EHsc) +# Separate functions for linker. +add_compile_options(/Gy) +# Conformance: wchar_t is a native type, not a typedef. +add_compile_options(/Zc:wchar_t) +# Use the __cdecl calling convention for all functions. +add_compile_options(/Gd) + +# Disable incremental linking. +string(APPEND win_link_flags " /INCREMENTAL:NO") +# Remove dead code. +string(APPEND win_link_flags " /OPT:REF") +# Allow executables to be randomly rebased at load time (enables virtual address allocation randomization). +string(APPEND win_link_flags " /DYNAMICBASE") +# Executable is compatible with the Windows Data Execution Prevention. +string(APPEND win_link_flags " /NXCOMPAT") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${win_link_flags}") diff --git a/src/third_party/wiredtiger/cmake/helpers.cmake b/src/third_party/wiredtiger/cmake/helpers.cmake new file mode 100644 index 00000000000..f3019627b8e --- /dev/null +++ b/src/third_party/wiredtiger/cmake/helpers.cmake @@ -0,0 +1,648 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +include(CheckIncludeFiles) +include(CheckSymbolExists) +include(CheckLibraryExists) +include(CheckTypeSize) + +# Helper function for evaluating a list of dependencies. Mostly used by the +# "config_X" helpers to evaluate the dependencies required to enable the config +# option. +# depends - a list (semicolon seperated) of dependencies to evaluate. +# enabled - name of the output variable set with either 'ON' or 'OFF' (based +# on evaluated dependencies). Output variable is set in the callers scope. +function(eval_dependency depends enabled) + # If no dependencies were given then we default to an enabled state. + if(("${depends}" STREQUAL "") OR ("${depends}" STREQUAL "NOTFOUND")) + set(enabled ON PARENT_SCOPE) + return() + endif() + # Evaluate each dependency. + set(is_enabled ON) + foreach(dependency ${depends}) + string(REGEX REPLACE " +" ";" dependency "${dependency}") + if(NOT (${dependency})) + set(is_enabled OFF) + break() + endif() + endforeach() + set(enabled ${is_enabled} PARENT_SCOPE) +endfunction() + +# config_string(config_name description DEFAULT [DEPENDS ] [INTERNAL]) +# Defines a string configuration option. The configuration option is stored in the cmake cache +# and can be exported to the wiredtiger config header. +# config_name - name of the configuration option. +# description - docstring to describe the configuration option (viewable in the cmake-gui). +# DEFAULT - Default value of the configuration string. Used when not manually set +# by a cmake script or in the cmake-gui. +# DEPENDS - list of dependencies (semicolon seperated) required for the configuration string +# to be present and set in the cache. If any of the dependencies aren't met, the +# configuration value won't be present in the cache. +# INTERNAL - hides the configuration option from the cmake-gui by default. Useful if you don't want +# to expose the variable by default to the user i.e keep it internal to the implementation +# (but still need it in the cache). +function(config_string config_name description) + cmake_parse_arguments( + PARSE_ARGV + 2 + "CONFIG_STR" + "INTERNAL" + "DEFAULT;DEPENDS" + "" + ) + + if (NOT "${CONFIG_STR_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to config_str: ${CONFIG_STR_UNPARSED_ARGUMENTS}") + endif() + + # Check that the configs dependencies are enabled before setting it to a visible enabled state. + eval_dependency("${CONFIG_STR_DEPENDS}" enabled) + set(default_value "${CONFIG_STR_DEFAULT}") + if(enabled) + # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state + # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. + if(${config_name}_DISABLED) + unset(${config_name}_DISABLED CACHE) + set(${config_name} ${default_value} CACHE STRING "${description}" FORCE) + else() + set(${config_name} ${default_value} CACHE STRING "${description}") + endif() + if (CONFIG_STR_INTERNAL) + # Mark as an advanced variable, hiding it from initial UI's views. + mark_as_advanced(FORCE ${config_name}) + endif() + else() + # Config doesn't meet dependency requirements, remove it from the cache and flag it as disabled. + unset(${config_name} CACHE) + set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) + endif() +endfunction() + +# config_choice(config_name description OPTIONS ) +# Defines a configuration option, bounded with pre-set toggleable values. The configuration option is stored +# in the cmake cache and can be exported to the wiredtiger config header. We default to the first *available* option in the +# list if the config has not been manually set by a cmake script or in the cmake-gui. +# config_name - name of the configuration option. +# description - docstring to describe the configuration option (viewable in the cmake-gui). +# OPTIONS - a list of option values that the configuration option can be set to. Each option is itself a semicolon +# seperated list consisting of ";;". +# * option-name: name of the given option stored in the ${config_name} cache variable and presented +# to users in the gui (usually something understandable). +# * config-name: an additional cached configuration variable that is made available if the option is selected. +# It is only present if the option is chosen, otherwise it is unset. +# * option-dependencies: dependencies required for the option to be made available. If its dependencies aren't met +# the given option will become un-selectable. +function(config_choice config_name description) + cmake_parse_arguments( + PARSE_ARGV + 2 + "CONFIG_OPT" + "" + "" + "OPTIONS" + ) + + if (NOT "${CONFIG_OPT_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to config_opt: ${CONFIG_OPT_UNPARSED_ARGUMENTS}") + endif() + # We require option values (not optional) + if ("${CONFIG_OPT_OPTIONS}" STREQUAL "") + message(FATAL_ERROR "No options passed") + endif() + + set(found_option ON) + set(found_pre_set OFF) + set(default_config_field "") + set(default_config_var "") + foreach(curr_option ${CONFIG_OPT_OPTIONS}) + list(LENGTH curr_option opt_length) + if (NOT opt_length EQUAL 3) + message(FATAL_ERROR "Invalid option format: ${curr_option}") + endif() + # We expect three items defined for each option. + list(GET curr_option 0 option_config_field) + list(GET curr_option 1 option_config_var) + list(GET curr_option 2 option_depends) + # Check that the options dependencies are enabled before setting it to a selectable state. + eval_dependency("${option_depends}" enabled) + if(enabled) + list(APPEND all_option_config_fields ${option_config_field}) + # The first valid/selectable option found will be the default config value. + if (found_option) + set(found_option OFF) + set(default_config_field "${option_config_field}") + set(default_config_var "${option_config_var}") + endif() + # Check if the option is already set with this given field. We don't want to override the configs value + # with a default value if its already been pre-set in the cache e.g. by early config scripts. + if("${${config_name}}" STREQUAL "${option_config_field}") + set(${option_config_var} ON CACHE INTERNAL "" FORCE) + set(${config_name}_CONFIG_VAR ${option_config_var} CACHE INTERNAL "" FORCE) + set(found_pre_set ON) + set(found_option OFF) + set(default_config_field "${option_config_field}") + set(default_config_var "${option_config_var}") + else() + # Clear the cache of the current set value. + set(${option_config_var} OFF CACHE INTERNAL "" FORCE) + endif() + else() + unset(${option_config_var} CACHE) + # Check if the option is already set with this given field - we want to clear it if so. + if ("${${config_name}_CONFIG_VAR}" STREQUAL "${option_config_var}") + unset(${config_name}_CONFIG_VAR CACHE) + endif() + if("${${config_name}}" STREQUAL "${option_config_field}") + unset(${config_name} CACHE) + endif() + endif() + endforeach() + # If the config hasn't been set we can load it with the default option found earlier. + if(NOT found_pre_set) + set(${default_config_var} ON CACHE INTERNAL "" FORCE) + set(${config_name} ${default_config_field} CACHE STRING ${description}) + set(${config_name}_CONFIG_VAR ${default_config_var} CACHE INTERNAL "" FORCE) + endif() + set_property(CACHE ${config_name} PROPERTY STRINGS ${all_option_config_fields}) +endfunction() + +# config_bool(config_name description DEFAULT [DEPENDS ] [DEPENDS_ERROR ]) +# Defines a boolean (ON/OFF) configuration option . The configuration option is stored in the cmake cache +# and can be exported to the wiredtiger config header. +# config_name - name of the configuration option. +# description - docstring to describe the configuration option (viewable in the cmake-gui). +# DEFAULT - default value of the configuration bool (ON/OFF). Used when not manually set +# by a cmake script or in the cmake-gui or when dependencies aren't met. +# DEPENDS - list of dependencies (semicolon seperated) required for the configuration bool +# to be set to the desired value. If any of the dependencies aren't met the configuration value +# will be set to its default value. +# DEPENDS_ERROR - specifically throw a fatal error when the configuration option is set to +# despite failing on its dependencies. This is mainly used for commandline-like options where you want +# to signal a specific error to the caller when dependencies aren't met e.g. toolchain is missing library (as opposed to +# silently defaulting). +function(config_bool config_name description) + cmake_parse_arguments( + PARSE_ARGV + 2 + "CONFIG_BOOL" + "" + "DEFAULT;DEPENDS" + "DEPENDS_ERROR" + ) + + if(NOT "${CONFIG_BOOL_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to config_bool: ${CONFIG_BOOL_UNPARSED_ARGUMENTS}") + endif() + # We require a default value (not optional). + if("${CONFIG_BOOL_DEFAULT}" STREQUAL "") + message(FATAL_ERROR "No default value passed") + endif() + + set(depends_err_value) + set(depends_err_message "") + # If DEPENDS_ERROR is specifically set, parse the value we want to throw an error on if the dependency fails. + if(CONFIG_BOOL_DEPENDS_ERROR) + list(LENGTH CONFIG_BOOL_DEPENDS_ERROR depends_error_length) + if(NOT depends_error_length EQUAL 2) + message(FATAL_ERROR "Invalid usage of DEPENDS_ERROR: requires ") + else() + list(GET CONFIG_BOOL_DEPENDS_ERROR 0 err_val) + if(err_val) + set(depends_err_value "1") + else() + set(depends_err_value "0") + endif() + list(GET CONFIG_BOOL_DEPENDS_ERROR 1 depends_err_message) + endif() + endif() + + # Check that the configs dependencies are enabled before setting it to a visible enabled state. + eval_dependency("${CONFIG_BOOL_DEPENDS}" enabled) + if(enabled) + # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. + # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. + if(${config_name}_DISABLED) + unset(${config_name}_DISABLED CACHE) + set(${config_name} ${CONFIG_BOOL_DEFAULT} CACHE STRING "${description}" FORCE) + else() + set(${config_name} ${CONFIG_BOOL_DEFAULT} CACHE STRING "${description}") + endif() + else() + set(config_value "0") + if (${${config_name}}) + set(config_value "1") + endif() + # If the user tries to set the config option to a given value when its dependencies + # are not met, throw an error (when DEPENDS_ERROR is explicitly set). + if(CONFIG_BOOL_DEPENDS_ERROR) + if(${depends_err_value} EQUAL ${config_value}) + message(FATAL_ERROR "Unable to set ${config_name}: ${depends_err_message}") + endif() + endif() + # Config doesn't meet dependency requirements, set its default state and flag it as disabled. + set(${config_name} OFF CACHE STRING "${description}" FORCE) + set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) + endif() +endfunction() + +# config_func(config_name description FUNC FILE [DEPENDS ] [LIBS ]) +# Defines a boolean (0/1) configuration option based on whether a given function symbol exists. +# The configuration option is stored in the cmake cache and can be exported to the wiredtiger config header. +# config_name - name of the configuration option. +# description - docstring to describe the configuration option (viewable in the cmake-gui). +# FUNC - function symbol we want to search for. +# FILE - header we expect the function symbol to be defined e.g a std header. +# DEPENDS - list of dependencies (semicolon seperated) required for the configuration to be evaluated. +# If any of the dependencies aren't met the configuration value will be set to '0' (false). +# LIBS - a list of any additional library dependencies needed to successfully link with the function symbol. +function(config_func config_name description) + cmake_parse_arguments( + PARSE_ARGV + 2 + "CONFIG_FUNC" + "" + "FUNC;DEPENDS;FILES;LIBS" + "" + ) + + if (NOT "${CONFIG_FUNC_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to config_func: ${CONFIG_FUNC_UNPARSED_ARGUMENTS}") + endif() + # We require an include header (not optional). + if ("${CONFIG_FUNC_FILES}" STREQUAL "") + message(FATAL_ERROR "No file list passed") + endif() + # We require a function symbol (not optional). + if ("${CONFIG_FUNC_FUNC}" STREQUAL "") + message(FATAL_ERROR "No function passed") + endif() + + # Check that the configs dependencies are enabled before setting it to a visible enabled state. + eval_dependency("${CONFIG_FUNC_DEPENDS}" enabled) + if(enabled) + set(CMAKE_REQUIRED_LIBRARIES "${CONFIG_FUNC_LIBS}") + if((NOT "${WT_ARCH}" STREQUAL "") AND (NOT "${WT_ARCH}" STREQUAL "")) + # 'check_symbol_exists' won't use our current cache when test compiling the function symbol. + # To get around this we need to ensure we manually forward WT_ARCH and WT_OS as a minimum. This is particularly + # needed if 'check_symbol_exists' will leverage one of our toolchain files. + set(CMAKE_REQUIRED_FLAGS "-DWT_ARCH=${WT_ARCH} -DWT_OS=${WT_OS}") + endif() + check_symbol_exists(${CONFIG_FUNC_FUNC} "${CONFIG_FUNC_FILES}" has_symbol_${config_name}) + set(CMAKE_REQUIRED_LIBRARIES) + set(CMAKE_REQUIRED_FLAGS) + set(has_symbol "0") + if(has_symbol_${config_name}) + set(has_symbol ${has_symbol_${config_name}}) + endif() + # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. + # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. + if(${config_name}_DISABLED) + unset(${config_name}_DISABLED CACHE) + set(${config_name} ${has_symbol} CACHE STRING "${description}" FORCE) + else() + set(${config_name} ${has_symbol} CACHE STRING "${description}") + endif() + # 'check_symbol_exists' sets our given temp variable into the cache. Clear this so it doesn't persist between + # configuration runs. + unset(has_symbol_${config_name} CACHE) + else() + # Config doesn't meet dependency requirements, set a disabled state. + set(${config_name} 0 CACHE INTERNAL "" FORCE) + set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) + endif() +endfunction() + + +# config_include(config_name description FILE [DEPENDS ]) +# Defines a boolean (0/1) configuration option based on whether a given include header exists. +# The configuration option is stored in the cmake cache and can be exported to the wiredtiger config header. +# config_name - name of the configuration option. +# description - docstring to describe the configuration option (viewable in the cmake-gui). +# FILE - header we want to search for e.g a std header. +# DEPENDS - list of dependencies (semicolon seperated) required for the configuration to be evaluated. +# If any of the dependencies aren't met the configuration value will be set to '0' (false). +function(config_include config_name description) + cmake_parse_arguments( + PARSE_ARGV + 2 + "CONFIG_INCLUDE" + "" + "FILE;DEPENDS" + "" + ) + + if (NOT "${CONFIG_INCLUDE_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to config_include: ${CONFIG_INCLUDE_UNPARSED_ARGUMENTS}") + endif() + # We require a include header (not optional). + if ("${CONFIG_INCLUDE_FILE}" STREQUAL "") + message(FATAL_ERROR "No include file passed") + endif() + + # Check that the configs dependencies are enabled before setting it to a visible enabled state. + eval_dependency("${CONFIG_INCLUDE_DEPENDS}" enabled) + if(enabled) + # 'check_include_files' won't use our current cache when test compiling the include header. + # To get around this we need to ensure we manually forward WT_ARCH and WT_OS as a minimum. This is particularly + # needed if 'check_include_files' will leverage one of our toolchain files. + if((NOT "${WT_ARCH}" STREQUAL "") AND (NOT "${WT_ARCH}" STREQUAL "")) + set(CMAKE_REQUIRED_FLAGS "-DWT_ARCH=${WT_ARCH} -DWT_OS=${WT_OS}") + endif() + check_include_files(${CONFIG_INCLUDE_FILE} has_include_${config_name}) + set(CMAKE_REQUIRED_FLAGS) + set(has_include "0") + if(has_include_${config_name}) + set(has_include ${has_include_${config_name}}) + endif() + # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. + # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. + if(${config_name}_DISABLED) + unset(${config_name}_DISABLED CACHE) + set(${config_name} ${has_include} CACHE STRING "${description}" FORCE) + else() + set(${config_name} ${has_include} CACHE STRING "${description}") + endif() + # 'check_include_files' sets our given temp variable into the cache. Clear this so it doesn't persist between + # configuration runs. + unset(has_include_${config_name} CACHE) + else() + set(${config_name} OFF CACHE INTERNAL "" FORCE) + set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) + endif() + # Set an internal cahce variable with the CPP include statement. We can use this when building out our config header. + if (${${config_name}}) + set(${config_name}_DECL "#include <${CONFIG_INCLUDE_FILE}>" CACHE INTERNAL "") + endif() +endfunction() + +# config_lib(config_name description LIB FUNC [DEPENDS ] [HEADER ]) +# Defines a boolean (0/1) configuration option based on whether a given library exists. +# The configuration option is stored in the cmake cache and can be exported to the wiredtiger config header. +# config_name - name of the configuration option. +# description - docstring to describe the configuration option (viewable in the cmake-gui). +# LIB - library we are searching for (defined as if we are linking against it e.g -lpthread). +# FUNC - function symbol we expect to be available to link against within the library. +# DEPENDS - list of dependencies (semicolon seperated) required for the configuration to be evaluated. +# If any of the dependencies aren't met the configuration value will be set to '0' (false). +function(config_lib config_name description) + cmake_parse_arguments( + PARSE_ARGV + 2 + "CONFIG_LIB" + "" + "LIB;DEPENDS;HEADER" + "" + ) + + if (NOT "${CONFIG_LIB_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to config_lib: ${CONFIG_LIB_UNPARSED_ARGUMENTS}") + endif() + # We require a library (not optional). + if ("${CONFIG_LIB_LIB}" STREQUAL "") + message(FATAL_ERROR "No library passed") + endif() + + # Check that the configs dependencies are enabled before setting it to a visible enabled state. + eval_dependency("${CONFIG_LIB_DEPENDS}" enabled) + if(enabled) + message("-- Looking for library ${CONFIG_LIB_LIB}") + # 'check_library_exists' won't use our current cache when test compiling the library. + # To get around this we need to ensure we manually forward WT_ARCH and WT_OS as a minimum. This is particularly + # needed if 'check_library_exists' will leverage one of our toolchain files. + if((NOT "${WT_ARCH}" STREQUAL "") AND (NOT "${WT_ARCH}" STREQUAL "")) + set(CMAKE_REQUIRED_FLAGS "-DWT_ARCH=${WT_ARCH} -DWT_OS=${WT_OS}") + endif() + find_library(has_lib_${config_name} ${CONFIG_LIB_LIB}) + set(CMAKE_REQUIRED_FLAGS) + set(has_lib "0") + if(has_lib_${config_name}) + set(has_lib ${has_lib_${config_name}}) + if (CONFIG_LIB_HEADER) + find_path(include_path_${config_name} ${CONFIG_LIB_HEADER}) + if (include_path_${config_name}) + message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}, include path ${include_path_${config_name}}") + include_directories(${include_path_${config_name}}) + else() + message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}") + endif() + unset(include_path_${config_name} CACHE) + else() + message("-- Looking for library ${CONFIG_LIB_LIB}: found ${has_lib_${config_name}}") + endif() + else() + message("-- Looking for library ${CONFIG_LIB_LIB}: NOT found") + endif() + # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. + # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. + if(${config_name}_DISABLED) + unset(${config_name}_DISABLED CACHE) + set(${config_name} ${has_lib} CACHE STRING "${description}" FORCE) + else() + set(${config_name} ${has_lib} CACHE STRING "${description}") + endif() + # 'check_library_exists' sets our given temp variable into the cache. Clear this so it doesn't persist between + # configuration runs. + unset(has_lib_${config_name} CACHE) + else() + message("-- Not looking for library ${CONFIG_LIB_LIB}: disabled") + set(${config_name} 0 CACHE INTERNAL "" FORCE) + set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) + endif() +endfunction() + +# config_compile(config_name description SOURCE [DEPENDS ] [LIBS ]) +# Defines a boolean (0/1) configuration option based on whether a source file can be successfully compiled and run. Used +# to determine if more fine grained functionality is supported on a given target environment (beyond what function +# symbols, libraries and headers are available). The configuration option is stored in the cmake cache and can be +# exported to the wiredtiger config header. +# config_name - name of the configuration option. +# description - docstring to describe the configuration option (viewable in the cmake-gui). +# SOURCE - specific source file we want to test compile. +# DEPENDS - list of dependencies (semicolon seperated) required for the configuration to be evaluated. +# If any of the dependencies aren't met the configuration value will be set to '0' (false). +# LIBS - a list of any additional library dependencies needed to successfully compile the source. +function(config_compile config_name description) + cmake_parse_arguments( + PARSE_ARGV + 2 + "CONFIG_COMPILE" + "" + "SOURCE;DEPENDS;LIBS" + "" + ) + + if (NOT "${CONFIG_COMPILE_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to config_compile: ${CONFIG_COMPILE_UNPARSED_ARGUMENTS}") + endif() + # We require a source file (not optional). + if ("${CONFIG_COMPILE_SOURCE}" STREQUAL "") + message(FATAL_ERROR "No source passed") + endif() + + # Check that the configs dependencies are enabled before setting it to a visible enabled state. + eval_dependency("${CONFIG_COMPILE_DEPENDS}" enabled) + if(enabled) + # Test compile the source file. + try_run( + can_run_${config_name} can_compile_${config_name} + ${CMAKE_CURRENT_BINARY_DIR} + ${CONFIG_COMPILE_SOURCE} + CMAKE_FLAGS "-DWT_ARCH=${WT_ARCH}" "-DWT_OS=${WT_OS}" + LINK_LIBRARIES "${CONFIG_COMPILE_LIBS}" + ) + set(can_run "0") + if((NOT "${can_run_${config_name}}" STREQUAL "FAILED_TO_RUN") AND + ("${can_run_${config_name}}" STREQUAL "0")) + set(can_run "1") + endif() + # Set an internal cache variable "${config_name}_DISABLED" to capture its enabled/disabled state. + # We want to ensure we capture a transition from a disabled to enabled state when dependencies are met. + if(${config_name}_DISABLED) + unset(${config_name}_DISABLED CACHE) + set(${config_name} ${can_run} CACHE STRING "${description}" FORCE) + else() + set(${config_name} ${can_run} CACHE STRING "${description}") + endif() + # 'try_run' sets our given temp variable into the cache. Clear this so it doesn't persist between + # configuration runs. + unset(can_run_${config_name} CACHE) + unset(can_compile_${config_name} CACHE) + else() + set(${config_name} 0 CACHE INTERNAL "" FORCE) + set(${config_name}_DISABLED ON CACHE INTERNAL "" FORCE) + endif() +endfunction() + +# test_type_size(type output_size) +# Helper function that tests for a given types size and returns its value if found. +# type - name of the type to test. +# output_size - name of the output variable, set with either the types size or "" (empty string) +# if not found. +# EXTRA_INCLUDES - extra/optional include files to access the given type e.g. a custom typedef in an include header. +function(test_type_size type output_size) + cmake_parse_arguments( + PARSE_ARGV + 2 + "TEST_TYPE" + "" + "" + "EXTRA_INCLUDES" + ) + + if (NOT "${TEST_TYPE_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to assert_type: ${TEST_TYPE_UNPARSED_ARGUMENTS}") + endif() + + set(CMAKE_EXTRA_INCLUDE_FILES "${TEST_TYPE_EXTRA_INCLUDES}") + check_type_size(${type} TEST_TYPE) + set(CMAKE_EXTRA_INCLUDE_FILES) + + if(NOT HAVE_TEST_TYPE) + set(${output_size} "" PARENT_SCOPE) + else() + set(${output_size} ${TEST_TYPE} PARENT_SCOPE) + endif() +endfunction() + +# assert_type_size(type size) +# Wrapper function around 'test_type_size' that additionally asserts whether the given types meets an expected size. +# Throws a fatal error if the type is not found or doesn't equal the expected size. +# type - eame of the type to test. +# size - expected size of the type. +# EXTRA_INCLUDES - extra/optional include files to access the given type e.g. a custom typedef in an include header. +function(assert_type_size type size) + cmake_parse_arguments( + PARSE_ARGV + 2 + "ASSERT_TYPE" + "" + "" + "EXTRA_INCLUDES" + ) + + if (NOT "${ASSERT_TYPE_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "Unknown arguments to assert_type: ${ASSERT_TYPE_UNPARSED_ARGUMENTS}") + endif() + + set(additional_args "") + if(${ASSERT_TYPE_EXTRA_INCLUDES}) + set(additional_args "EXTRA_INCLUDES ${ASSERT_TYPE_EXTRA_INCLUDES}") + endif() + test_type_size(${type} output_type_size ${additional_args}) + + if(${output_type_size} EQUAL "") + # Type does not exist. + message(FATAL_ERROR "Type assertion failed: ${type} does not exists") + endif() + + if((NOT ${size} EQUAL 0) AND (NOT ${output_type_size} EQUAL ${size})) + # Type does not meet size assertion. + message(FATAL_ERROR "Type assertion failed: ${type} does not equal size ${size}") + endif() +endfunction() + +# parse_filelist_source(filelist output_var) +# A helper function that parses the list of sources (usually found in "dist/filelist"). This returning a list of +# sources that can then be used to generate the necessary build rules for the wiredtiger library. Additionally +# uses the config values "WT_ARCH" and "WT_OS" when extracting platform specific sources from the file list. +# filelist - Destination of 'filelist' file. +# output_var - name of the output variable that will be set with the parsed sources. Output variable is set in +# the callers scope. +function(parse_filelist_source filelist output_var) + set(arch_host "") + set(plat_host "") + # Determine architecture host for our filelist parse. + if(WT_X86) + set(arch_host "X86_HOST") + elseif(WT_AARCH64) + set(arch_host "ARM64_HOST") + elseif(WT_PPC64) + set(arch_host "POWERPC_HOST") + elseif(WT_S390X) + set(arch_host "ZSERIES_HOST") + endif() + # Determine platform host for our filelist parse. + if(WT_POSIX) + set(plat_host "POSIX_HOST") + elseif(WT_WIN) + set(plat_host "WINDOWS_HOST") + endif() + + # Read file list and parse into list. + file(READ "${filelist}" contents NEWLINE_CONSUME) + string(REGEX REPLACE "\n" ";" contents "${contents}") + set(output_files "") + foreach(file ${contents}) + if(${file} MATCHES "^#.*$") + continue() + endif() + string(REGEX REPLACE "[ \t\r]+" ";" file_contents ${file}) + list(LENGTH file_contents file_contents_len) + if (file_contents_len EQUAL 1) + list(APPEND output_files ${file}) + elseif(file_contents_len EQUAL 2) + list(GET file_contents 0 file_name) + list(GET file_contents 1 file_group) + if ((${file_group} STREQUAL "${plat_host}") OR (${file_group} STREQUAL "${arch_host}")) + list(APPEND output_files ${file_name}) + get_filename_component(file_ext ${file_name} EXT) + # POWERPC and ZSERIES hosts use the '.sx' extension for their ASM files. We need to + # manually tell CMake to ASM compile these files otherwise it will ignore them during + # compilation process. + if("${file_ext}" STREQUAL ".sx") + set_source_files_properties(${file_name} PROPERTIES LANGUAGE ASM) + endif() + endif() + else() + message(FATAL_ERROR "filelist (${filelist}) has an unexpected format [Invalid Line: \"${file}]\"") + endif() + endforeach() + set(${output_var} ${output_files} PARENT_SCOPE) +endfunction() diff --git a/src/third_party/wiredtiger/cmake/install/install.cmake b/src/third_party/wiredtiger/cmake/install/install.cmake new file mode 100644 index 00000000000..ec9b30234d6 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/install/install.cmake @@ -0,0 +1,37 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information +# + +include(GNUInstallDirs) + +# Library installs + +# Define the public headers for wiredtiger library to be used when installing the target. +set_property( + TARGET wiredtiger + PROPERTY PUBLIC_HEADER + ${CMAKE_BINARY_DIR}/include/wiredtiger.h + ${CMAKE_SOURCE_DIR}/src/include/wiredtiger_ext.h +) +# Set the version property of the wiredtiger library so we can export a versioned install. +set_target_properties(wiredtiger PROPERTIES VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") + +# Install the wiredtiger library target. +install(TARGETS wiredtiger + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) + +# Create our wiredtiger pkgconfig (for POSIX builds). +if(WT_POSIX) + configure_file(${CMAKE_CURRENT_LIST_DIR}/wiredtiger.pc.in wiredtiger.pc @ONLY) + install( + FILES ${CMAKE_BINARY_DIR}/wiredtiger.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig + ) +endif() diff --git a/src/third_party/wiredtiger/cmake/install/wiredtiger.pc.in b/src/third_party/wiredtiger/cmake/install/wiredtiger.pc.in new file mode 100644 index 00000000000..9ed407b7fe4 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/install/wiredtiger.pc.in @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=@CMAKE_INSTALL_PREFIX@ +libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: WiredTiger +Description: The WiredTiger Data Engine +Requires: +Version: @VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_PATCH@ +Libs: -L${libdir} -lwiredtiger +Cflags: -I${includedir} diff --git a/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake b/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake new file mode 100644 index 00000000000..625bf9e88d5 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/strict/cl_strict.cmake @@ -0,0 +1,21 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +# Warning level 3. +list(APPEND win_c_flags "/WX") +# Ignore warning about mismatched const qualifiers. +list(APPEND win_c_flags "/wd4090") +# Ignore deprecated functions. +list(APPEND win_c_flags "/wd4996") +# Complain about unreferenced format parameter. +list(APPEND win_c_flags "/we4100") +# Enable security check. +list(APPEND win_c_flags "/GS") + +# Set our base compiler flags that can be used by the rest of our build. +set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${win_c_flags}" CACHE INTERNAL "" FORCE) diff --git a/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake b/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake new file mode 100644 index 00000000000..f87c988fe02 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/strict/clang_strict.cmake @@ -0,0 +1,65 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +list(APPEND clang_base_c_flags "-Weverything") +list(APPEND clang_base_c_flags "-Werror") +list(APPEND clang_base_c_flags "-Wno-cast-align") +list(APPEND clang_base_c_flags "-Wno-documentation-unknown-command") +list(APPEND clang_base_c_flags "-Wno-format-nonliteral") +list(APPEND clang_base_c_flags "-Wno-packed") +list(APPEND clang_base_c_flags "-Wno-padded") +list(APPEND clang_base_c_flags "-Wno-reserved-id-macro") +list(APPEND clang_base_c_flags "-Wno-zero-length-array") + +# We should turn on cast-qual, but not as a fatal error: see WT-2690. +# For now, turn it off. +list(APPEND clang_base_c_flags "-Wno-cast-qual") + +# Turn off clang thread-safety-analysis, it doesn't like some of the +# code patterns in WiredTiger. +list(APPEND clang_base_c_flags "-Wno-thread-safety-analysis") + +# On Centos 7.3.1611, system header files aren't compatible with +# -Wdisabled-macro-expansion. +list(APPEND clang_base_c_flags "-Wno-disabled-macro-expansion") + +# We occasionally use an extra semicolon to indicate an empty loop or +# conditional body. +list(APPEND clang_base_c_flags "-Wno-extra-semi-stmt") + +# Ignore unrecognized options. +list(APPEND clang_base_c_flags "-Wno-unknown-warning-option") + +if(WT_DARWIN AND (CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.1)) + # Apple clang has its own numbering system, and older OS X + # releases need some special love. Turn off some flags for + # Apple's clang 4.1: + # Apple clang version 4.1 + # ((tags/Apple/clang-421.11.66) (based on LLVM 3.1svn) + list(APPEND clang_base_c_flags "-Wno-attributes") + list(APPEND clang_base_c_flags "-Wno-pedantic") + list(APPEND clang_base_c_flags "-Wno-unused-command-line-argument") +endif() + +if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 10) + # Clang 10+ has added additional on-by-default diagnostics that isn't + # compatible with some of the code patterns in WiredTiger. + list(APPEND clang_base_c_flags "-Wno-implicit-fallthrough") + list(APPEND clang_base_c_flags "-Wno-implicit-int-float-conversion") +endif() + +if(WT_DARWIN AND NOT CMAKE_CROSSCOMPILING) + # If we are not cross-compiling, we can safely disable this diagnostic. + # Its incompatible with strict diagnostics when including external + # libraries that are not in the default linker path + # e.g. linking zlib/snappy/... from /usr/local/. + list(APPEND clang_base_c_flags "-Wno-poison-system-directories") +endif() + +# Set our base clang flags to ensure it propogates to the rest of our build. +set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${clang_base_c_flags}" CACHE INTERNAL "" FORCE) diff --git a/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake b/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake new file mode 100644 index 00000000000..420a4457676 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/strict/gcc_strict.cmake @@ -0,0 +1,79 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +list(APPEND gcc_base_c_flags "-Wall") +list(APPEND gcc_base_c_flags "-Wextra") +list(APPEND gcc_base_c_flags "-Werror") +list(APPEND gcc_base_c_flags "-Waggregate-return") +list(APPEND gcc_base_c_flags "-Wbad-function-cast") +list(APPEND gcc_base_c_flags "-Wcast-align") +list(APPEND gcc_base_c_flags "-Wdeclaration-after-statement") +list(APPEND gcc_base_c_flags "-Wdouble-promotion") +list(APPEND gcc_base_c_flags "-Wfloat-equal") +list(APPEND gcc_base_c_flags "-Wformat-nonliteral") +list(APPEND gcc_base_c_flags "-Wformat-security") +list(APPEND gcc_base_c_flags "-Wformat=2") +list(APPEND gcc_base_c_flags "-Winit-self") +list(APPEND gcc_base_c_flags "-Wjump-misses-init") +list(APPEND gcc_base_c_flags "-Wmissing-declarations") +list(APPEND gcc_base_c_flags "-Wmissing-field-initializers") +list(APPEND gcc_base_c_flags "-Wmissing-prototypes") +list(APPEND gcc_base_c_flags "-Wnested-externs") +list(APPEND gcc_base_c_flags "-Wold-style-definition") +list(APPEND gcc_base_c_flags "-Wpacked") +list(APPEND gcc_base_c_flags "-Wpointer-arith") +list(APPEND gcc_base_c_flags "-Wpointer-sign") +list(APPEND gcc_base_c_flags "-Wredundant-decls") +list(APPEND gcc_base_c_flags "-Wshadow") +list(APPEND gcc_base_c_flags "-Wsign-conversion") +list(APPEND gcc_base_c_flags "-Wstrict-prototypes") +list(APPEND gcc_base_c_flags "-Wswitch-enum") +list(APPEND gcc_base_c_flags "-Wundef") +list(APPEND gcc_base_c_flags "-Wuninitialized") +list(APPEND gcc_base_c_flags "-Wunreachable-code") +list(APPEND gcc_base_c_flags "-Wunused") +list(APPEND gcc_base_c_flags "-Wwrite-strings") + +# Non-fatal informational warnings. +# We don't turn on the unsafe-loop-optimizations warning after gcc7, +# it's too noisy to tolerate. Regardless, don't fail even when it's +# configured. +list(APPEND gcc_base_c_flags "-Wno-error=unsafe-loop-optimizations") +if(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 4.7) + list(APPEND gcc_base_c_flags "-Wno-c11-extensions") + list(APPEND gcc_base_c_flags "-Wunsafe-loop-optimizations") +elseif(${CMAKE_C_COMPILER_VERSION} VERSION_EQUAL 5) + list(APPEND gcc_base_c_flags "-Wunsafe-loop-optimizations") +endif() + +if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 5) + list(APPEND gcc_base_c_flags "-Wformat-signedness") + list(APPEND gcc_base_c_flags "-Wjump-misses-init") + list(APPEND gcc_base_c_flags "-Wredundant-decls") + list(APPEND gcc_base_c_flags "-Wunused-macros") + list(APPEND gcc_base_c_flags "-Wvariadic-macros") +endif() +if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 6) + list(APPEND gcc_base_c_flags "-Wduplicated-cond") + list(APPEND gcc_base_c_flags "-Wlogical-op") + list(APPEND gcc_base_c_flags "-Wunused-const-variable=2") +endif() +if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 7) + list(APPEND gcc_base_c_flags "-Walloca") + list(APPEND gcc_base_c_flags "-Walloc-zero") + list(APPEND gcc_base_c_flags "-Wduplicated-branches") + list(APPEND gcc_base_c_flags "-Wformat-overflow=2") + list(APPEND gcc_base_c_flags "-Wformat-truncation=2") + list(APPEND gcc_base_c_flags "-Wrestrict") +endif() +if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 8) + list(APPEND gcc_base_c_flags "-Wmultistatement-macros") +endif() + +# Set our base gcc flags to ensure it propogates to the rest of our build. +set(COMPILER_DIAGNOSTIC_FLAGS "${COMPILER_DIAGNOSTIC_FLAGS};${gcc_base_c_flags}" CACHE INTERNAL "" FORCE) diff --git a/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_clang.cmake new file mode 100644 index 00000000000..98cc2a8892b --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_clang.cmake @@ -0,0 +1,15 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "aarch64-unknown-linux-gnu") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_gcc.cmake new file mode 100644 index 00000000000..b82bbfe9b94 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/aarch64/linux/plat_gcc.cmake @@ -0,0 +1,11 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PREFIX "aarch64-linux-gnu-") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/cl.cmake b/src/third_party/wiredtiger/cmake/toolchains/cl.cmake new file mode 100644 index 00000000000..d6467238ce3 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/cl.cmake @@ -0,0 +1,13 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +cmake_minimum_required(VERSION 3.10.0) + +set(CMAKE_C_COMPILER "cl.exe") +set(CMAKE_CXX_COMPILER "cl.exe") +set(CMAKE_ASM_COMPILER "cl.exe") diff --git a/src/third_party/wiredtiger/cmake/toolchains/clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/clang.cmake new file mode 100644 index 00000000000..5b789f1134f --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/clang.cmake @@ -0,0 +1,36 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +cmake_minimum_required(VERSION 3.10.0) + +set(CMAKE_C_COMPILER "clang") +set(CMAKE_C_COMPILER_ID "Clang") + +set(CMAKE_CXX_COMPILER "clang++") +set(CMAKE_CXX_COMPILER_ID "Clang++") + +set(CMAKE_ASM_COMPILER "clang") +set(CMAKE_ASM_COMPILER_ID "Clang") + +if(NOT "${COMPILE_DEFINITIONS}" STREQUAL "") + ### Additional check to overcome check_[symbol|include|function]_exits using toolchain file without passing WT_ARCH and WT_OS. + string(REGEX MATCH "-DWT_ARCH=([A-Za-z0-9]+) -DWT_OS=([A-Za-z0-9]+)" _ ${COMPILE_DEFINITIONS}) + set(wt_config_arch ${CMAKE_MATCH_1}) + set(wt_config_os ${CMAKE_MATCH_2}) +else() + set(wt_config_arch ${WT_ARCH}) + set(wt_config_os ${WT_OS}) +endif() + +# Include any platform specific clang configurations and flags e.g. target-tuple, flags. +if((NOT "${wt_config_arch}" STREQUAL "") AND (NOT "${wt_config_os}" STREQUAL "")) + if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/${wt_config_arch}/${wt_config_os}/plat_clang.cmake") + message(FATAL_ERROR "(${wt_config_arch}/${wt_config_os}) directory does not have a plat_clang.cmake file") + endif() + include("${CMAKE_CURRENT_LIST_DIR}/${wt_config_arch}/${wt_config_os}/plat_clang.cmake") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/gcc.cmake new file mode 100644 index 00000000000..0fa64ed0037 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/gcc.cmake @@ -0,0 +1,31 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +cmake_minimum_required(VERSION 3.10.0) + +if(NOT "${COMPILE_DEFINITIONS}" STREQUAL "") + ### Additional check to overcome check_[symbol|include|function]_exits using toolchain file without passing WT_ARCH and WT_OS. + string(REGEX MATCH "-DWT_ARCH=([A-Za-z0-9]+) -DWT_OS=([A-Za-z0-9]+)" _ ${COMPILE_DEFINITIONS}) + set(wt_config_arch ${CMAKE_MATCH_1}) + set(wt_config_os ${CMAKE_MATCH_2}) +else() + set(wt_config_arch ${WT_ARCH}) + set(wt_config_os ${WT_OS}) +endif() + +# Include any platform specific gcc configurations and flags e.g. target-tuple, flags. +if((NOT "${wt_config_arch}" STREQUAL "") AND (NOT "${wt_config_os}" STREQUAL "")) + if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/${wt_config_arch}/${wt_config_os}/plat_gcc.cmake") + message(FATAL_ERROR "(${wt_config_arch}/${wt_config_os}) directory does not have a plat_gcc.cmake file") + endif() + include("${CMAKE_CURRENT_LIST_DIR}/${wt_config_arch}/${wt_config_os}/plat_gcc.cmake") +endif() + +set(CMAKE_C_COMPILER "${CROSS_COMPILER_PREFIX}gcc") +set(CMAKE_CXX_COMPILER "${CROSS_COMPILER_PREFIX}g++") +set(CMAKE_ASM_COMPILER "${CROSS_COMPILER_PREFIX}gcc") diff --git a/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_clang.cmake new file mode 100644 index 00000000000..2353ba1b7f7 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_clang.cmake @@ -0,0 +1,17 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +cmake_minimum_required(VERSION 3.10.0) + +if(NOT TOOLCHAIN_ROOT) + set(TOOLCHAIN_ROOT "/opt/mongodbtoolchain/v3") +endif() + +set(CMAKE_C_COMPILER "${TOOLCHAIN_ROOT}/bin/clang") +set(CMAKE_CXX_COMPILER "${TOOLCHAIN_ROOT}/bin/clang++") +set(CMAKE_ASM_COMPILER "${TOOLCHAIN_ROOT}/bin/clang") diff --git a/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_gcc.cmake new file mode 100644 index 00000000000..136d4a33f4f --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/mongodbtoolchain_v3_gcc.cmake @@ -0,0 +1,17 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +cmake_minimum_required(VERSION 3.10.0) + +if(NOT TOOLCHAIN_ROOT) + set(TOOLCHAIN_ROOT "/opt/mongodbtoolchain/v3") +endif() + +set(CMAKE_C_COMPILER "${TOOLCHAIN_ROOT}/bin/gcc") +set(CMAKE_CXX_COMPILER "${TOOLCHAIN_ROOT}/bin/g++") +set(CMAKE_ASM_COMPILER "${TOOLCHAIN_ROOT}/bin/gcc") diff --git a/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_clang.cmake new file mode 100644 index 00000000000..f03d4fdbd97 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_clang.cmake @@ -0,0 +1,15 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "powerpc64le-unknown-linux-gnu") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_gcc.cmake new file mode 100644 index 00000000000..47e706c0d39 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/ppc64le/linux/plat_gcc.cmake @@ -0,0 +1,11 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PEFIX "powerpc64le-linux-gnu-") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_clang.cmake new file mode 100644 index 00000000000..f643b9c8df8 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_clang.cmake @@ -0,0 +1,15 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "s390x-ibm-linux-gnu") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_gcc.cmake new file mode 100644 index 00000000000..abfff342dfb --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/s390x/linux/plat_gcc.cmake @@ -0,0 +1,11 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PEFIX "s390x-linux-gnu-") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_clang.cmake new file mode 100644 index 00000000000..e7130be256f --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_clang.cmake @@ -0,0 +1,15 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "x86_64-apple-darwin") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_gcc.cmake new file mode 100644 index 00000000000..7a463e46d96 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/x86/darwin/plat_gcc.cmake @@ -0,0 +1,14 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +# We are not cross-compiling if our system is Darwin, hence the "x86_64-apple-darwin-" +# prefix is not necessary when we are not cross-compiling. Just default to the host +# installed 'gcc' binary. +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PREFIX "x86_64-apple-darwin-") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_clang.cmake b/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_clang.cmake new file mode 100644 index 00000000000..126a3a1bbbe --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_clang.cmake @@ -0,0 +1,15 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(TRIPLE_TARGET "x86_64-pc-linux-gnu") + set(CROSS_COMPILER_PREFIX ${TRIPLE_TARGET}-) + set(CMAKE_C_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_CXX_COMPILER_TARGET "${TRIPLE_TARGET}") + set(CMAKE_ASM_COMPILER_TARGET "${TRIPLE_TARGET}") +endif() diff --git a/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_gcc.cmake b/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_gcc.cmake new file mode 100644 index 00000000000..2b36fb667d5 --- /dev/null +++ b/src/third_party/wiredtiger/cmake/toolchains/x86/linux/plat_gcc.cmake @@ -0,0 +1,11 @@ +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILER_PREFIX "x86_64-linux-gnu-") +endif() diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 9663188789b..4b4c98f6018 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1432,10 +1432,11 @@ methods = { selects a simple hexadecimal format, "json" selects a JSON format with each record formatted as fields named by column names if available, "pretty" selects a human-readable format (making it - incompatible with the "load") and "print" selects a format where only - non-printing characters are hexadecimal encoded. These formats are - compatible with the @ref util_dump and @ref util_load commands''', - choices=['hex', 'json', 'pretty', 'print']), + incompatible with the "load"), "pretty_hex" is similar to "pretty" (also incompatible with + "load") except raw byte data elements will be printed like "hex" format, and + "print" selects a format where only non-printing characters are hexadecimal encoded. These + formats are compatible with the @ref util_dump and @ref util_load commands''', + choices=['hex', 'json', 'pretty', 'pretty_hex', 'print']), Config('incremental', '', r''' configure the cursor for block incremental backup usage. These formats are only compatible with the backup data source; see @ref backup''', @@ -1659,10 +1660,11 @@ methods = { applicable only for prepared transactions. Indicates if the prepare timestamp and the commit timestamp of this transaction can be rounded up. If the prepare timestamp is less than the oldest - timestamp, the prepare timestamp will be rounded to the oldest + timestamp, the prepare timestamp will be rounded to the oldest timestamp. If the commit timestamp is less than the prepare timestamp, the commit timestamp will be rounded up to the prepare - timestamp''', type='boolean'), + timestamp. Allows setting the prepared timestamp smaller than or equal + to the latest active read timestamp''', type='boolean'), Config('read', 'false', r''' if the read timestamp is less than the oldest timestamp, the read timestamp will be rounded up to the oldest timestamp''', diff --git a/src/third_party/wiredtiger/dist/s_copyright.list b/src/third_party/wiredtiger/dist/s_copyright.list index f1c879a555b..8f4534c2670 100644 --- a/src/third_party/wiredtiger/dist/s_copyright.list +++ b/src/third_party/wiredtiger/dist/s_copyright.list @@ -1,6 +1,6 @@ skip bench/workgen/workgen/workgen.py skip bench/workgen/workgen_wrap.cxx -skip build_cmake/install/wiredtiger.pc.in +skip cmake/install/wiredtiger.pc.in skip build_win/wiredtiger_config.h skip dist/api_config.py skip dist/api_config_gen.py diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index 83b31aa1a09..8b5c02aa8a8 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -57,7 +57,7 @@ wtperf_config() glossarychk() { (cd ../src/docs && grep '^' arch-glossary.dox | cut -d'>' -f3 | sed -e 's/<.*//') > $t - sort --ignore-case $t > $t2 + sort -f $t > $t2 if ! diff $t $t2; then echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" echo "arch-glossary.dox entries out of order" diff --git a/src/third_party/wiredtiger/dist/s_export b/src/third_party/wiredtiger/dist/s_export index 0df4f16b3ef..8615f1415b2 100755 --- a/src/third_party/wiredtiger/dist/s_export +++ b/src/third_party/wiredtiger/dist/s_export @@ -51,7 +51,7 @@ check() # ../build_posix/.libs/libwiredtiger.$ext (autotools build) # ../.libs/libwiredtiger.$ext (autotools build at top level) # ../build/libwiredtiger.$ext (cmake build) -# ../build_cmake/libwiredtiger.$ext (cmake build) +# ../cmake_build/libwiredtiger.$ext (cmake build) found=0 for f in $(find .. -name "libwiredtiger.*" -print); do diff --git a/src/third_party/wiredtiger/dist/s_string b/src/third_party/wiredtiger/dist/s_string index 4aab8a16f38..c41f0d82381 100755 --- a/src/third_party/wiredtiger/dist/s_string +++ b/src/third_party/wiredtiger/dist/s_string @@ -32,7 +32,8 @@ replace() { check() { # Strip out git hashes, which are seven character hex strings. # Strip out double quote char literals ('"'), they confuse aspell. - sed -e 's/ [0-9a-f]\{7\} / /g' -e "s/'\"'//g" ../$2 | + # Strip out calls to __wt_getopt so the option lists don't have to be spelling words. + sed -e 's/ [0-9a-f]\{7\} / /g' -e "s/'\"'//g" -e 's/__wt_getopt([^()]*)//' ../$2 | aspell --lang=en_US $1 list | sort -u | comm -23 /dev/stdin s_string.ok > $t diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index d0b2ab55adf..1b4cf50716e 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -113,7 +113,6 @@ Decrement Decrypt DeleteFileW Destructor -Dh EACCES EAGAIN EB @@ -156,7 +155,6 @@ FindClose FindFirstFile FindNextFileW Fixup -Fk FlushFileBuffers Fprintf FreeBSD @@ -244,7 +242,6 @@ LevelDB Levyx LibFuzzer LmRrSVv -LmsT LoadLoad LockFile Lookaside @@ -498,7 +495,6 @@ abcdef abcdefghijklmnopqrstuvwxyz addl addr -af agc alfred alloc @@ -587,7 +583,6 @@ ccc ccr cd ce -ceh celsius centric cfg @@ -962,7 +957,6 @@ iter iteratively iters jjj -jnr jprx json kb @@ -1036,7 +1030,6 @@ lwsync lx lz lzo -mT madvise majmin majorp @@ -1159,8 +1152,6 @@ osfhandle other's ovfl ownp -pR -pS packv pagedump pagesize @@ -1218,7 +1209,6 @@ pv pvA pwrite py -qRrT qdown qqq qrrSS @@ -1279,7 +1269,6 @@ runtime rwlock sH sHQ -sT sanitizer sanitizers scalability @@ -1472,7 +1461,6 @@ util utils uu vN -vW va valgrind valuep @@ -1493,10 +1481,7 @@ vsize vsnprintf vtype vunpack -vw vxr -vxz -vz waitpid waker wakeup diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 32d3d237858..f35e56252a1 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -876,6 +876,7 @@ conn_dsrc_stats = [ ########################################## TxnStat('txn_checkpoint_obsolete_applied', 'transaction checkpoints due to obsolete pages'), TxnStat('txn_read_race_prepare_update', 'race to read prepared update retry'), + TxnStat('txn_rts_delete_rle_skipped', 'rollback to stable skipping delete rle'), TxnStat('txn_rts_hs_removed', 'rollback to stable updates removed from history store'), TxnStat('txn_rts_hs_restore_updates', 'rollback to stable restored updates from history store'), TxnStat('txn_rts_hs_restore_tombstones', 'rollback to stable restored tombstones from history store'), @@ -883,6 +884,7 @@ conn_dsrc_stats = [ TxnStat('txn_rts_inconsistent_ckpt', 'rollback to stable inconsistent checkpoint'), TxnStat('txn_rts_keys_removed', 'rollback to stable keys removed'), TxnStat('txn_rts_keys_restored', 'rollback to stable keys restored'), + TxnStat('txn_rts_stable_rle_skipped', 'rollback to stable skipping stable rle'), TxnStat('txn_rts_sweep_hs_keys', 'rollback to stable sweeping history store keys'), TxnStat('txn_update_conflict', 'update conflicts'), ] diff --git a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c index eeca2a83b99..11384645222 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c +++ b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c @@ -83,9 +83,10 @@ typedef struct { /* * Statistics are collected but not yet exposed. */ - uint64_t fh_ops; /* Non-read/write operations in file handles */ - uint64_t object_flushes; /* (What would be) writes to the cloud */ - uint64_t op_count; /* Number of operations done on local */ + uint64_t fh_ops; /* Non-read/write operations in file handles */ + uint64_t object_writes; /* (What would be) writes to the cloud */ + uint64_t object_reads; /* (What would be) reads to the cloud */ + uint64_t op_count; /* Number of operations done on local */ uint64_t read_ops; uint64_t write_ops; @@ -129,7 +130,8 @@ static int local_file_copy( LOCAL_STORAGE *, WT_SESSION *, const char *, const char *, WT_FS_OPEN_FILE_TYPE); static int local_get_directory(const char *, ssize_t len, char **); static int local_path(WT_FILE_SYSTEM *, const char *, const char *, char **); -static int local_writeable(LOCAL_STORAGE *, const char *name, bool *writeable); +static int local_stat( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, bool, struct stat *); /* * Forward function declarations for storage source API implementation @@ -243,10 +245,12 @@ local_delay(LOCAL_STORAGE *local) int ret; ret = 0; - if (local->force_delay != 0 && local->object_flushes % local->force_delay == 0) { + if (local->force_delay != 0 && + (local->object_reads + local->object_writes) % local->force_delay == 0) { WT_VERBOSE_LS(local, - "Artificial delay %" PRIu32 " milliseconds after %" PRIu64 " object flushes\n", - local->delay_ms, local->object_flushes); + "Artificial delay %" PRIu32 " milliseconds after %" PRIu64 " object reads, %" PRIu64 + " object writes\n", + local->delay_ms, local->object_reads, local->object_writes); /* * tv_usec has type suseconds_t, which is signed (hence the s), but ->delay_ms is unsigned. * In both gcc8 and gcc10 with -Wsign-conversion enabled (as we do) this causes a spurious @@ -257,9 +261,11 @@ local_delay(LOCAL_STORAGE *local) tv.tv_usec = (suseconds_t)(local->delay_ms % 1000) * 1000; (void)select(0, NULL, NULL, NULL, &tv); } - if (local->force_error != 0 && local->object_flushes % local->force_error == 0) { - WT_VERBOSE_LS(local, "Artificial error returned after %" PRIu64 " object flushes\n", - local->object_flushes); + if (local->force_error != 0 && + (local->object_reads + local->object_writes) % local->force_error == 0) { + WT_VERBOSE_LS(local, + "Artificial error returned after %" PRIu64 " object reads, %" PRIu64 " object writes\n", + local->object_reads, local->object_writes); ret = ENETUNREACH; } @@ -316,28 +322,6 @@ local_get_directory(const char *s, ssize_t len, char **copy) return (ret); } -/* - * local_writeable -- - * Check if a file can be written, or equivalently, check to see that it has not been flushed. - * This will be true if it is in the regular file system (not one managed by local_store). - */ -static int -local_writeable(LOCAL_STORAGE *local, const char *name, bool *writeablep) -{ - struct stat sb; - int ret; - - ret = 0; - *writeablep = false; - - if (stat(name, &sb) == 0) - *writeablep = true; - else if (errno != ENOENT) - ret = local_err(local, NULL, errno, "%s: stat", name); - - return (ret); -} - /* * local_bucket_path -- * Construct the bucket pathname from the file system and local name. @@ -387,6 +371,49 @@ local_path(WT_FILE_SYSTEM *file_system, const char *dir, const char *name, char return (ret); } +/* + * local_stat -- + * Perform the stat system call for a name in the file system. + */ +static int +local_stat(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, const char *caller, + bool must_exist, struct stat *statp) +{ + int ret; + char *path; + + path = NULL; + + /* + * We check to see if the file exists in the cache first, and if not the bucket directory. This + * maps what a real cloud implementation would do. This will allow us to instrument this code to + * try out and measure caching implementations. + */ + if ((ret = local_cache_path(file_system, name, &path)) != 0) + goto err; + + ret = stat(path, statp); + if (ret != 0 && errno == ENOENT) { + /* It's not in the cache, try the bucket directory. */ + free(path); + if ((ret = local_bucket_path(file_system, name, &path)) != 0) + goto err; + ret = stat(path, statp); + } + if (ret != 0) { + /* + * If the file must exist, report the error no matter what. + */ + if (must_exist || errno != ENOENT) + ret = local_err(WT_FS2LOCAL(file_system), session, errno, "%s: %s stat", path, caller); + else + ret = errno; + } +err: + free(path); + return (ret); +} + /* * local_add_reference -- * Add a reference to the storage source so we can reference count to know when to really @@ -517,34 +544,16 @@ local_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, struct stat sb; LOCAL_STORAGE *local; int ret; - char *path; local = WT_FS2LOCAL(file_system); - path = NULL; - - /* If the file exists directly in the file system, it's not yet flushed, and we're done. */ - ret = stat(name, &sb); - if (ret == 0) { - *existp = true; - return (0); - } else if (errno != ENOENT) - ret = local_err(local, session, errno, "%s: ss_exist stat", path); - local->op_count++; - if ((ret = local_cache_path(file_system, name, &path)) != 0) - goto err; + *existp = false; - ret = stat(path, &sb); - if (ret == 0) + if ((ret = local_stat(file_system, session, name, "ss_exist", false, &sb)) == 0) *existp = true; - else if (errno == ENOENT) { + else if (ret == ENOENT) ret = 0; - *existp = false; - } else - ret = local_err(local, session, errno, "%s: ss_exist stat", path); -err: - free(path); return (ret); } @@ -636,7 +645,7 @@ local_flush(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, WT_FILE_SYST if ((ret = local_file_copy(local, session, source, dest_path, WT_FS_OPEN_FILE_TYPE_DATA)) != 0) goto err; - local->object_flushes++; + local->object_writes++; err: free(dest_path); @@ -781,14 +790,14 @@ local_directory_list_internal(WT_FILE_SYSTEM *file_system, WT_SESSION *session, *countp = 0; /* - * We list items in the cache directory (these have 'finished' flushing). + * The items in the bucket directory represent the definitive list. */ - if ((dirp = opendir(local_fs->cache_dir)) == NULL) { + if ((dirp = opendir(local_fs->bucket_dir)) == NULL) { ret = errno; if (ret == 0) ret = EINVAL; return ( - local_err(local, session, ret, "%s: ss_directory_list: opendir", local_fs->cache_dir)); + local_err(local, session, ret, "%s: ss_directory_list: opendir", local_fs->bucket_dir)); } for (count = 0; (dp = readdir(dirp)) != NULL && (limit == 0 || count < limit);) { @@ -868,9 +877,7 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, WT_FILE_SYSTEM *wt_fs; struct stat sb; int ret; - char *alloced_path; - const char *path; - bool create, exists; + char *bucket_path, *cache_path; (void)flags; /* Unused */ @@ -880,7 +887,11 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, local_fs = (LOCAL_FILE_SYSTEM *)file_system; local = local_fs->local_storage; wt_fs = local_fs->wt_fs; - alloced_path = NULL; + bucket_path = cache_path = NULL; + + if ((flags & WT_FS_OPEN_READONLY) == 0 || (flags & WT_FS_OPEN_CREATE) != 0) + return ( + local_err(local, session, EINVAL, "ss_open_object: readonly access required: %s", name)); /* * We expect that the local file system will be used narrowly, like when creating or opening a @@ -902,41 +913,32 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, ret = ENOMEM; goto err; } - create = ((flags & WT_FS_OPEN_CREATE) != 0); - if (!create) { - ret = stat(name, &sb); - if (ret != 0 && errno != ENOENT) { - ret = local_err(local, session, errno, "%s: local_open stat", name); + if ((ret = local_cache_path(file_system, name, &cache_path)) != 0) + goto err; + ret = stat(cache_path, &sb); + if (ret != 0) { + if (errno != ENOENT) { + ret = local_err(local, session, errno, "%s: local_open stat", cache_path); goto err; } - exists = (ret == 0); - } else - exists = false; - if (create || exists) - /* The file has not been flushed, use the file directly in the file system. */ - path = name; - else { - if ((ret = local_cache_path(file_system, name, &alloced_path)) != 0) + + /* + * The file doesn't exist locally, make a copy of it from the cloud. + */ + if ((ret = local_bucket_path(file_system, name, &bucket_path)) != 0) goto err; - path = alloced_path; - ret = stat(path, &sb); - if (ret != 0 && errno != ENOENT) { - ret = local_err(local, session, errno, "%s: local_open stat", path); + + if ((ret = local_delay(local)) != 0) + goto err; + + if ((ret = local_file_copy( + local, session, bucket_path, cache_path, WT_FS_OPEN_FILE_TYPE_DATA)) != 0) goto err; - } - exists = (ret == 0); - } - /* - * TODO: tiered: If the file doesn't exist locally, make a copy of it from the cloud here. - * - */ -#if 0 - if ((flags & WT_FS_OPEN_READONLY) != 0 && !exists) { - } -#endif - if ((ret = wt_fs->fs_open_file(wt_fs, session, path, file_type, flags, &wt_fh)) != 0) { - ret = local_err(local, session, ret, "ss_open_object: open: %s", path); + local->object_reads++; + } + if ((ret = wt_fs->fs_open_file(wt_fs, session, cache_path, file_type, flags, &wt_fh)) != 0) { + ret = local_err(local, session, ret, "ss_open_object: open: %s", name); goto err; } local_fh->fh = wt_fh; @@ -985,7 +987,8 @@ local_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, WT_SHOW_STRING(local_fh->fh->name)); err: - free(alloced_path); + free(bucket_path); + free(cache_path); if (ret != 0) { if (local_fh != NULL) local_file_close_internal(local, session, local_fh); @@ -995,75 +998,30 @@ err: /* * local_rename -- - * POSIX rename, for files not yet flushed to the cloud. If a file has been flushed, we don't - * support this operation. That is because cloud implementations may not support it, and more - * importantly, we consider anything in the cloud to be readonly as far as the custom file - * system is concerned. + * POSIX rename, not supported for cloud objects. */ static int local_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *from, const char *to, uint32_t flags) { - LOCAL_FILE_SYSTEM *local_fs; - LOCAL_STORAGE *local; - WT_FILE_SYSTEM *wt_fs; - int ret; - bool writeable; - - local = WT_FS2LOCAL(file_system); - local_fs = (LOCAL_FILE_SYSTEM *)file_system; - wt_fs = local_fs->wt_fs; - - local->op_count++; - if ((ret = local_writeable(local, from, &writeable)) != 0) - goto err; - if (!writeable) { - ret = local_err(local, session, ENOTSUP, "%s: rename of flushed file not allowed", from); - goto err; - } - - if ((ret = wt_fs->fs_rename(wt_fs, session, from, to, flags)) != 0) { - ret = local_err(local, session, ret, "fs_rename"); - goto err; - } + (void)to; /* unused */ + (void)flags; /* unused */ -err: - return (ret); + return (local_err( + WT_FS2LOCAL(file_system), session, ENOTSUP, "%s: rename of file not supported", from)); } /* * local_remove -- - * POSIX remove, for files not yet flushed to the cloud. If a file has been flushed, we don't - * support this operation. We consider anything in the cloud to be readonly as far as the custom - * file system is concerned. + * POSIX remove, not supported for cloud objects. */ static int local_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, uint32_t flags) { - LOCAL_STORAGE *local; - int ret; - bool writeable; - - (void)flags; /* Unused */ - - local = WT_FS2LOCAL(file_system); + (void)flags; /* unused */ - local->op_count++; - if ((ret = local_writeable(local, name, &writeable)) != 0) - goto err; - if (!writeable) { - ret = local_err(local, session, ENOTSUP, "%s: remove of flushed file not allowed", name); - goto err; - } - - ret = unlink(name); - if (ret != 0) { - ret = local_err(local, session, errno, "%s: ss_remove unlink", name); - goto err; - } - -err: - return (ret); + return (local_err( + WT_FS2LOCAL(file_system), session, ENOTSUP, "%s: remove of file not supported", name)); } /* @@ -1076,31 +1034,16 @@ local_size(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, w struct stat sb; LOCAL_STORAGE *local; int ret; - char *path; local = WT_FS2LOCAL(file_system); - path = NULL; - local->op_count++; + *sizep = 0; - /* If the file exists directly in the file system, it's not yet flushed, so use it */ - ret = stat(name, &sb); - if (ret == ENOENT) { - /* Otherwise, we'll see if it's in the cache directory. */ - if ((ret = local_cache_path(file_system, name, &path)) != 0) - goto err; - - ret = stat(path, &sb); - /* TODO: tiered: if we still get an ENOENT, then we'd need to ping the cloud to get the - * size. */ - } - if (ret == 0) - *sizep = sb.st_size; - else - ret = local_err(local, session, errno, "%s: ss_size stat", path); + if ((ret = local_stat(file_system, session, name, "ss_size", true, &sb)) != 0) + goto err; + *sizep = sb.st_size; err: - free(path); return (ret); } @@ -1249,14 +1192,10 @@ local_file_size(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *size static int local_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) { - LOCAL_FILE_HANDLE *local_fh; - WT_FILE_HANDLE *wt_fh; - - local_fh = (LOCAL_FILE_HANDLE *)file_handle; - wt_fh = local_fh->fh; - - local_fh->local->fh_ops++; - return (wt_fh->fh_sync(wt_fh, session)); + /* This is a no-op. We could also disallow it. */ + (void)file_handle; + (void)session; + return (0); } /* @@ -1268,13 +1207,14 @@ local_file_write( WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, const void *buf) { LOCAL_FILE_HANDLE *local_fh; - WT_FILE_HANDLE *wt_fh; - local_fh = (LOCAL_FILE_HANDLE *)file_handle; - wt_fh = local_fh->fh; + (void)offset; + (void)len; + (void)buf; - local_fh->local->write_ops++; - return (wt_fh->fh_write(wt_fh, session, offset, len, buf)); + local_fh = (LOCAL_FILE_HANDLE *)file_handle; + return (local_err(local_fh->local, session, ENOTSUP, "ss_open_object: write not supported: %s", + local_fh->iface.name)); } /* diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 0d24bead828..52e281c34f8 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "ffebd365d6c3c18422437bcae51d8a26a17772a0" + "commit": "b385c984870fec2d693dece8e79876fd5e8bf867" } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 32a6172cde6..ff4c436a6e1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -522,8 +522,8 @@ __cursor_key_order_check_row(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo __wt_msg(session, "WT_CURSOR.%s out-of-order returns: returned key %.1024s then key %.1024s", next ? "next" : "prev", __wt_buf_set_printable_format( - session, cbt->lastkey->data, cbt->lastkey->size, btree->key_format, a), - __wt_buf_set_printable_format(session, key->data, key->size, btree->key_format, b))); + session, cbt->lastkey->data, cbt->lastkey->size, btree->key_format, false, a), + __wt_buf_set_printable_format(session, key->data, key->size, btree->key_format, false, b))); WT_ERR(__wt_msg(session, "dumping the cursor page")); WT_ERR(__wt_debug_cursor_page(&cbt->iface, NULL)); WT_ERR_PANIC(session, EINVAL, "found key out-of-order returns"); diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index ef8e42f97e5..c952272bb8b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -944,7 +944,7 @@ __curfile_update_check(WT_CURSOR_BTREE *cbt) else if (btree->type != BTREE_COL_VAR) return (0); - return (__wt_txn_update_check(session, cbt, upd, NULL)); + return (__wt_txn_modify_check(session, cbt, upd, NULL)); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 81f419184f5..d718239a575 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -146,7 +146,7 @@ __debug_item_value(WT_DBG *ds, const char *tag, const void *data_arg, size_t siz if (session->dump_raw) return (ds->f(ds, "\t%s%s{%s}\n", tag == NULL ? "" : tag, tag == NULL ? "" : " ", - __wt_buf_set_printable(session, data_arg, size, ds->t1))); + __wt_buf_set_printable(session, data_arg, size, false, ds->t1))); /* * If the format is 'S', it's a string and our version of it may not yet be nul-terminated. @@ -157,7 +157,7 @@ __debug_item_value(WT_DBG *ds, const char *tag, const void *data_arg, size_t siz size = ds->t2->size + 1; } return (ds->f(ds, "\t%s%s{%s}\n", tag == NULL ? "" : tag, tag == NULL ? "" : " ", - __wt_buf_set_printable_format(session, data_arg, size, ds->value_format, ds->t1))); + __wt_buf_set_printable_format(session, data_arg, size, ds->value_format, false, ds->t1))); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_misc.c b/src/third_party/wiredtiger/src/btree/bt_misc.c index 6488ac7148a..9d60ec73ce8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_misc.c +++ b/src/third_party/wiredtiger/src/btree/bt_misc.c @@ -91,7 +91,7 @@ __wt_key_string( #ifdef HAVE_DIAGNOSTIC if (session->dump_raw) - return (__wt_buf_set_printable(session, data_arg, size, buf)); + return (__wt_buf_set_printable(session, data_arg, size, false, buf)); #endif /* @@ -107,7 +107,7 @@ __wt_key_string( size = sizeof(WT_ERR_STRING); } } - return (__wt_buf_set_printable_format(session, data_arg, size, key_format, buf)); + return (__wt_buf_set_printable_format(session, data_arg, size, key_format, false, buf)); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 8cb00cbde81..cc8c8d47803 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -646,10 +646,11 @@ __slvg_trk_leaf(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, uint8_t *ad __wt_verbose(session, WT_VERB_SALVAGE, "%s start key %s", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, ss->tmp1), - __wt_buf_set_printable(session, trk->row_start.data, trk->row_start.size, ss->tmp2)); + __wt_buf_set_printable( + session, trk->row_start.data, trk->row_start.size, false, ss->tmp2)); __wt_verbose(session, WT_VERB_SALVAGE, "%s stop key %s", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, ss->tmp1), - __wt_buf_set_printable(session, trk->row_stop.data, trk->row_stop.size, ss->tmp2)); + __wt_buf_set_printable(session, trk->row_stop.data, trk->row_stop.size, false, ss->tmp2)); /* Row-store pages can contain overflow items. */ WT_ERR(__slvg_trk_leaf_ovfl(session, dsk, trk)); @@ -1879,7 +1880,7 @@ __slvg_row_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref, WT_S break; __wt_verbose(session, WT_VERB_SALVAGE, "%s merge discarding leading key %s", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, ss->tmp1), - __wt_buf_set_printable(session, key->data, key->size, ss->tmp2)); + __wt_buf_set_printable(session, key->data, key->size, false, ss->tmp2)); ++skip_start; } if (F_ISSET(trk, WT_TRACK_CHECK_STOP)) @@ -1894,7 +1895,7 @@ __slvg_row_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref, WT_S break; __wt_verbose(session, WT_VERB_SALVAGE, "%s merge discarding trailing key %s", __wt_addr_string(session, trk->trk_addr, trk->trk_addr_size, ss->tmp1), - __wt_buf_set_printable(session, key->data, key->size, ss->tmp2)); + __wt_buf_set_printable(session, key->data, key->size, false, ss->tmp2)); ++skip_stop; } diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 764cec570c2..27d1d90b6d7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -625,8 +625,8 @@ __verify_row_int_key_order( " on the page at %s sorts before the last key appearing on page %s, earlier in the tree: " "%s, %s", entry, __verify_addr_string(session, ref, vs->tmp1), (char *)vs->max_addr->data, - __wt_buf_set_printable(session, item.data, item.size, vs->tmp2), - __wt_buf_set_printable(session, vs->max_key->data, vs->max_key->size, vs->tmp3)); + __wt_buf_set_printable(session, item.data, item.size, false, vs->tmp2), + __wt_buf_set_printable(session, vs->max_key->data, vs->max_key->size, false, vs->tmp3)); /* Update the largest key we've seen to the key just checked. */ WT_RET(__wt_buf_set(session, vs->max_key, item.data, item.size)); @@ -677,8 +677,9 @@ __verify_row_leaf_key_order(WT_SESSION_IMPL *session, WT_REF *ref, WT_VSTUFF *vs "the first key on the page at %s sorts equal to or less than the last key appearing " "on the page at %s, earlier in the tree: %s, %s", __verify_addr_string(session, ref, vs->tmp2), (char *)vs->max_addr->data, - __wt_buf_set_printable(session, vs->tmp1->data, vs->tmp1->size, vs->tmp3), - __wt_buf_set_printable(session, vs->max_key->data, vs->max_key->size, vs->tmp4)); + __wt_buf_set_printable(session, vs->tmp1->data, vs->tmp1->size, false, vs->tmp3), + __wt_buf_set_printable( + session, vs->max_key->data, vs->max_key->size, false, vs->tmp4)); } /* Update the largest key we've seen to the last key on this page. */ @@ -804,7 +805,7 @@ __verify_key_hs( WT_ERR_MSG(session, WT_ERROR, "key %s has a overlap of timestamp ranges between history store stop timestamp %s " "being newer than a more recent timestamp range having start timestamp %s", - __wt_buf_set_printable(session, tmp1->data, tmp1->size, vs->tmp2), + __wt_buf_set_printable(session, tmp1->data, tmp1->size, false, vs->tmp2), __wt_timestamp_to_string(older_stop_ts, ts_string[0]), __wt_timestamp_to_string(newer_start_ts, ts_string[1])); } diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index 994e35806f2..d25699df070 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -249,8 +249,9 @@ __verify_row_key_order_check(WT_SESSION_IMPL *session, WT_ITEM *last, uint32_t l ret = WT_ERROR; WT_ERR_VRFY(session, "the %" PRIu32 " and %" PRIu32 " keys on page at %s are incorrectly sorted: %s, %s", - last_cell_num, cell_num, tag, __wt_buf_set_printable(session, last->data, last->size, tmp1), - __wt_buf_set_printable(session, current->data, current->size, tmp2)); + last_cell_num, cell_num, tag, + __wt_buf_set_printable(session, last->data, last->size, false, tmp1), + __wt_buf_set_printable(session, current->data, current->size, false, tmp2)); err: __wt_scr_free(session, &tmp1); diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index 632526f83df..c14d3251042 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -35,15 +35,16 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; u_int i, skipdepth; - bool append, logged; + bool append, inserted_to_update_chain, logged; btree = CUR2BT(cbt); ins = NULL; page = cbt->ref->page; session = CUR2S(cbt); + last_upd = NULL; upd = upd_arg; prev_upd_ts = WT_TS_NONE; - append = logged = false; + append = inserted_to_update_chain = logged = false; /* * We should have one of the following: @@ -120,19 +121,15 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U } /* - * Delete, insert or update a column-store entry. - * - * If modifying a previously modified record, cursor.ins will be set to point to the correct - * update list. Create a new update entry and link it into the existing list. - * - * Else, allocate an insert array as necessary, build an insert/update structure pair, and link - * it into place. + * Modify a column-store entry. If modifying a previously modified record, cursor.ins will point + * to the correct update list; create a new update and link it into the already existing list. + * Otherwise, we have to insert a new insert/update pair into the column-store insert list. */ if (cbt->compare == 0 && cbt->ins != NULL) { old_upd = cbt->ins->upd; if (upd_arg == NULL) { - /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd, &prev_upd_ts)); + /* Make sure the modify can proceed. */ + WT_ERR(__wt_txn_modify_check(session, cbt, old_upd, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); @@ -142,7 +139,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_ERR(__wt_txn_modify(session, upd)); logged = true; - /* Avoid a data copy in WT_CURSOR.update. */ + /* Avoid WT_CURSOR.update data copy. */ __wt_upd_value_assign(cbt->modify_update, upd); } else { upd_size = __wt_update_list_memsize(upd); @@ -178,6 +175,10 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U /* Serialize the update. */ WT_ERR(__wt_update_serial(session, cbt, page, &cbt->ins->upd, &upd, upd_size, false)); } else { + /* Make sure the modify can proceed. */ + if (cbt->compare == 0 && upd_arg == NULL) + WT_ERR(__wt_txn_modify_check(session, cbt, NULL, NULL)); + /* Allocate the append/update list reference as necessary. */ if (append) { WT_PAGE_ALLOC_AND_SWAP(session, page, mod->mod_col_append, ins_headp, 1); @@ -218,7 +219,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_ERR(__wt_txn_modify(session, upd)); logged = true; - /* Avoid a data copy in WT_CURSOR.update. */ + /* Avoid WT_CURSOR.update data copy. */ __wt_upd_value_assign(cbt->modify_update, upd); } else upd_size = __wt_update_list_memsize(upd); @@ -254,12 +255,14 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U session, page, cbt->ins_head, cbt->ins_stack, &ins, ins_size, skipdepth, exclusive)); } + inserted_to_update_chain = true; + /* If the update was successful, add it to the in-memory log. */ if (logged && modify_type != WT_UPDATE_RESERVE) { WT_ERR(__wt_txn_log_op(session, cbt)); /* - * In case of append, the recno (key) for the value is assigned now. Set the recno in the + * In case of append, the recno (key) for the value is assigned now. Set the key in the * transaction operation to be used in case this transaction is prepared to retrieve the * update corresponding to this operation. */ @@ -268,14 +271,25 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U if (0) { err: - /* - * Remove the update from the current transaction, so we don't try to modify it on rollback. - */ + /* Remove the update from the current transaction; don't try to modify it on rollback. */ if (logged) __wt_txn_unmodify(session); + + /* Free any allocated insert list object. */ __wt_free(session, ins); - if (upd_arg == NULL) + + cbt->ins = NULL; + + /* Discard any allocated update, unless we failed after linking it into page memory. */ + if (upd_arg == NULL && !inserted_to_update_chain) __wt_free(session, upd); + + /* + * When prepending a list of updates to an update chain, we link them together; sever that + * link so our callers list doesn't point into page memory. + */ + if (last_upd != NULL) + last_upd->next = NULL; } return (ret); diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index c96b963cb1f..779f5112c12 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -109,8 +109,8 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, upd_entry = &cbt->ins->upd; if (upd_arg == NULL) { - /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd = *upd_entry, &prev_upd_ts)); + /* Make sure the modify can proceed. */ + WT_ERR(__wt_txn_modify_check(session, cbt, old_upd = *upd_entry, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); @@ -139,6 +139,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, upd_arg->next == NULL) || (upd_arg->type == WT_UPDATE_TOMBSTONE && upd_arg->next != NULL && upd_arg->next->type == WT_UPDATE_STANDARD && upd_arg->next->next == NULL)); + upd_size = __wt_update_list_memsize(upd); /* If there are existing updates, append them after the new updates. */ @@ -182,7 +183,6 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, * slot. That's hard, so we set a flag. */ WT_PAGE_ALLOC_AND_SWAP(session, page, mod->mod_row_insert, ins_headp, page->entries + 1); - ins_slot = F_ISSET(cbt, WT_CBT_SEARCH_SMALLEST) ? page->entries : cbt->slot; ins_headp = &mod->mod_row_insert[ins_slot]; @@ -218,6 +218,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, (upd_arg->type == WT_UPDATE_TOMBSTONE && upd_arg->next != NULL && upd_arg->next->type == WT_UPDATE_STANDARD && upd_arg->next->next == NULL) || (upd_arg->type == WT_UPDATE_STANDARD && upd_arg->next == NULL)); + upd_size = __wt_update_list_memsize(upd); } @@ -250,8 +251,10 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, inserted_to_update_chain = true; + /* If the update was successful, add it to the in-memory log. */ if (logged && modify_type != WT_UPDATE_RESERVE) { WT_ERR(__wt_txn_log_op(session, cbt)); + /* * Set the key in the transaction operation to be used in case this transaction is prepared * to retrieve the update corresponding to this operation. @@ -261,15 +264,23 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, if (0) { err: - /* - * Remove the update from the current transaction, so we don't try to modify it on rollback. - */ + /* Remove the update from the current transaction, don't try to modify it on rollback. */ if (logged) __wt_txn_unmodify(session); + + /* Free any allocated insert list object. */ __wt_free(session, ins); + cbt->ins = NULL; + + /* Discard any allocated update, unless we failed after linking it into page memory. */ if (upd_arg == NULL && !inserted_to_update_chain) __wt_free(session, upd); + + /* + * When prepending a list of updates to an update chain, we link them together; sever that + * link so our callers list doesn't point into page memory. + */ if (last_upd != NULL) last_upd->next = NULL; } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 693bf9aa554..a3d9af315a4 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -359,7 +359,10 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = { {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_wait", "boolean", NULL, NULL, NULL, 0}, {"debug", "category", NULL, NULL, confchk_WT_SESSION_open_cursor_debug_subconfigs, 1}, - {"dump", "string", NULL, "choices=[\"hex\",\"json\",\"pretty\",\"print\"]", NULL, 0}, + {"dump", "string", NULL, + "choices=[\"hex\",\"json\",\"pretty\",\"pretty_hex\"," + "\"print\"]", + NULL, 0}, {"incremental", "category", NULL, NULL, confchk_WT_SESSION_open_cursor_incremental_subconfigs, 7}, {"next_random", "boolean", NULL, NULL, NULL, 0}, {"next_random_sample_size", "string", NULL, NULL, NULL, 0}, diff --git a/src/third_party/wiredtiger/src/cursor/cur_bulk.c b/src/third_party/wiredtiger/src/cursor/cur_bulk.c index c23c1940cd6..5672cbbf1c7 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_bulk.c +++ b/src/third_party/wiredtiger/src/cursor/cur_bulk.c @@ -202,8 +202,8 @@ __bulk_row_keycmp_err(WT_CURSOR_BULK *cbulk) WT_ERR_MSG(session, EINVAL, "bulk-load presented with out-of-order keys: %s compares smaller than previously inserted " "key %s", - __wt_buf_set_printable(session, cursor->key.data, cursor->key.size, a), - __wt_buf_set_printable(session, cbulk->last.data, cbulk->last.size, b)); + __wt_buf_set_printable(session, cursor->key.data, cursor->key.size, false, a), + __wt_buf_set_printable(session, cbulk->last.data, cbulk->last.size, false, b)); err: __wt_scr_free(session, &a); diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 734acdbe70a..be4c1c7ef34 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -91,8 +91,8 @@ __curdump_get_key(WT_CURSOR *cursor, ...) WT_ERR(child->get_key(child, &item)); if (F_ISSET(cursor, WT_CURSTD_DUMP_PRETTY)) { - WT_IGNORE_RET_PTR(__wt_buf_set_printable_format( - session, item.data, item.size, cursor->key_format, &cursor->key)); + WT_IGNORE_RET_PTR(__wt_buf_set_printable_format(session, item.data, item.size, + cursor->key_format, F_ISSET(cursor, WT_CURSTD_DUMP_HEX), &cursor->key)); } else WT_ERR( __raw_to_dump(session, &item, &cursor->key, F_ISSET(cursor, WT_CURSTD_DUMP_HEX))); @@ -241,8 +241,8 @@ __curdump_get_value(WT_CURSOR *cursor, ...) WT_ERR(child->get_value(child, &item)); if (F_ISSET(cursor, WT_CURSTD_DUMP_PRETTY)) - WT_IGNORE_RET_PTR(__wt_buf_set_printable_format( - session, item.data, item.size, cursor->value_format, &cursor->value)); + WT_IGNORE_RET_PTR(__wt_buf_set_printable_format(session, item.data, item.size, + cursor->value_format, F_ISSET(cursor, WT_CURSTD_DUMP_HEX), &cursor->value)); else WT_ERR( __raw_to_dump(session, &item, &cursor->value, F_ISSET(cursor, WT_CURSTD_DUMP_HEX))); diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 26d6915724a..adacc1f3e68 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -1189,6 +1189,8 @@ __wt_cursor_init( dump_flag = WT_CURSTD_DUMP_PRINT; else if (WT_STRING_MATCH("pretty", cval.str, cval.len)) dump_flag = WT_CURSTD_DUMP_PRETTY; + else if (WT_STRING_MATCH("pretty_hex", cval.str, cval.len)) + dump_flag = WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRETTY; else dump_flag = WT_CURSTD_DUMP_HEX; F_SET(cursor, dump_flag); diff --git a/src/third_party/wiredtiger/src/docs/arch-hs.dox b/src/third_party/wiredtiger/src/docs/arch-hs.dox index e1d0a88c677..c6453108f03 100644 --- a/src/third_party/wiredtiger/src/docs/arch-hs.dox +++ b/src/third_party/wiredtiger/src/docs/arch-hs.dox @@ -89,7 +89,7 @@ be corrected. Consider the following update chain for a user table with btree id 1000 and data store key "AAA": -@plantuml_start{hs_update_chain.png} +@plantuml_start{hs_update_chain.png } @startuml{hs_update_chain.png} together { diff --git a/src/third_party/wiredtiger/src/docs/arch-log-file.dox b/src/third_party/wiredtiger/src/docs/arch-log-file.dox index a684e023d76..1ff828052db 100644 --- a/src/third_party/wiredtiger/src/docs/arch-log-file.dox +++ b/src/third_party/wiredtiger/src/docs/arch-log-file.dox @@ -1,7 +1,59 @@ /*! @arch_page arch-log-file Log File Format -The format of a WiredTiger log file is defined in \c log.h . -Each file begins with a fixed length header, followed by a set of -variable length log records. Though the records may have varying -length, they all begin on boundaries aligned to 128 bytes. +This page assumes familiarity with the logging subsystem. Please see +@ref arch-logging for the logging subsystem architecture description. + +Each log file begins with a fixed length header, followed by a set of +variable length log records. The fixed length header contains information +necessary for WiredTiger to determine that a file is a valid log file. +The header contains a magic number in the first 32 bits, the log version +number and the configured maximum log file size. + +When a new log file is created WiredTiger must synchronously write out the header before allowing +any user records to be written to the file. The content of the file header is +identical for every log file. This fact allows for pre-allocation of log files +so that user threads that happen to be writing out log records that require a log +file switch are usually not penalized with expensive I/O operations. The typical +log file switch is a rename instead. In addition to writing out the header, pre-allocation +also allocates the space for the maximum size of the log file so that the risk +of out of space errors is minimized. Pre-allocation is done with an internal +thread as described in @ref log_threads. + +Log files from log version 1 are generated by releases before WiredTiger version +3.0.0. In log version 1, user records start immediately after the header. +All log files from log version 2 onward write a system log record after +the header as the first record in the log file. Writing the system record +is part of creating the new log file. + +All records in the log, including the header, are subject to the minimum record +size. Typically that is 128 bytes. That value may be changed if direct I/O for +logging is turned on. When enabling direct I/O via \c direct_io=log to ::wiredtiger_open +then the minimum alignment is the value specified for \c buffer_alignment. +In all other cases it is 128 bytes. + +The reason that 128 bytes was chosen as the minimum record size was to eliminate any +possibility of false sharing in memory and invalidating cache lines when multiple +threads are writing their log records to the memory buffers in parallel. False sharing +would potentially impact performance. + +From version 2 onward the first record in the log file, typically at offset 128 in the +file, is a special system log +record that contains the LSN of the end of the previous log file. For the first +log file that value is an invalid LSN. For later log files the LSN indicates the end +of the previous file so that recovery can detect missing log records and holes in +log files that appear at the end of a file. + +As described in @ref log_subsystem, multiple threads are writing records to the log +in a lock free manner and multiple log buffer slots may be in flight at any given time. +A hole can be generated in a log file if a buffer with a later LSN is written before +a buffer with an earlier LSN. That can also happen at a log file boundary and that is why +knowing the LSN at the end of the previous log file is critical to recovery. + +The user can choose the maximum log file size via \c log=(file_max=size) configuration +to the ::wiredtiger_open call. Records written in the log are varying length +depending on the data written. In typical usage, the system will choose to switch +log files before writing a log buffer that exceeds the configured file size. However it +is possible that a single record itself could be larger than the configured file size. +In that case the system must allow that record to exceed the maximum file size. But it +will be the only record in that log file. */ diff --git a/src/third_party/wiredtiger/src/docs/build-posix.dox b/src/third_party/wiredtiger/src/docs/build-posix.dox index eb3528e249a..ea9bcb5ea90 100644 --- a/src/third_party/wiredtiger/src/docs/build-posix.dox +++ b/src/third_party/wiredtiger/src/docs/build-posix.dox @@ -141,19 +141,19 @@ configures WiredTiger to use gcc-based spinlocks). @section posix_compiler Changing compiler or loader options By default CMake will use your default system compiler (\c cc). If you want to use a specific toolchain you can pass a toolchain file. We have -provided a toolchain file for both GCC (\c build_cmake/toolchains/gcc.cmake) and Clang (\c build_cmake/toolchains/clang.cmake). To use either +provided a toolchain file for both GCC (\c cmake/toolchains/gcc.cmake) and Clang (\c cmake/toolchains/clang.cmake). To use either toolchain you can pass the \c -DCMAKE_TOOLCHAIN_FILE= to the CMake configuration step. For example: For example, to explicitly specify the GCC toolchain: @code -cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/gcc.cmake -G Ninja ../. +cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/gcc.cmake -G Ninja ../. @endcode To explicitly specify the Clang toolchain: @code -cmake -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/clang.cmake -G Ninja ../. +cmake -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -G Ninja ../. @endcode By default, WiredTiger builds with the \c -O3 compiler optimization flag diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index 4432ae32054..7d518c4a18e 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -187,7 +187,8 @@ format. @par -p Dump in human-readable format (pretty-print). The \c -p flag is incompatible -with the \c load command. +with the \c load command. The \c -p flag can be combined with \c -x. In this case, raw data elements +will be formatted like \c -x with hexadecimal encoding. @par -r Dump in reverse order, from largest key to smallest. @@ -199,7 +200,10 @@ timestamp. @par -x Dump all characters in a hexadecimal encoding (the default is to leave -printable characters unencoded). +printable characters unencoded). The \c -x flag can be combined with \c -p. In this case, the dump +will be formatted similar to \c -p except for raw data elements, which will look like \c -x with +hexadecimal encoding. If the two options are combined the output is no longer compatible +with \c load.
@section util_list wt list diff --git a/src/third_party/wiredtiger/src/docs/dump-formats.dox b/src/third_party/wiredtiger/src/docs/dump-formats.dox index 2a903f9c790..2a5a6475646 100644 --- a/src/third_party/wiredtiger/src/docs/dump-formats.dox +++ b/src/third_party/wiredtiger/src/docs/dump-formats.dox @@ -68,6 +68,7 @@ following information: @hrow{String, Meaning} @row{hex, the dumped data is in a hexadecimal dump format} @row{print, the dumped data is in a printable format} +@row{print hex, the dumped data is in a printable format with raw byte array elements printed same as in hex format} The dump header follows a single \c "Header" line in the file and @@ -77,10 +78,11 @@ string. The table or file can be recreated by calling WT_SESSION::create for each pair of lines in the header. The dump body follows a single \c "Data" line in the file and consists -of a text representation of the records in the table. Each record is a +of a text representation of the records in the table. Each record is represented by a pair of lines: the first line is the key and the second -line is the value. These lines are encoded in one of two formats: a -printable format and a hexadecimal format. +line is the value. These lines are encoded in one of the following formats: a +printable format, a hexadecimal format, and a printable format with raw byte array elements in +hexadecimal format. The printable format consists of literal printable characters, and hexadecimal encoded non-printable characters. Encoded characters are @@ -97,6 +99,9 @@ literal character is written as a pair of characters (first the high-nibble and then the low-nibble). For example, "0a" would be an ASCII newline character and "1b" would be an ASCII escape character. +The printable hexadecimal format is the same as the printable except all raw +byte array elements will be printed in hexadecimal format. + Because the definition of "printable" may depend on the application's locale, dump files in the printable output format may be less portable than dump files in the hexadecimal output format. diff --git a/src/third_party/wiredtiger/src/docs/schema.dox b/src/third_party/wiredtiger/src/docs/schema.dox index 57f2801884f..036411360c3 100644 --- a/src/third_party/wiredtiger/src/docs/schema.dox +++ b/src/third_party/wiredtiger/src/docs/schema.dox @@ -67,7 +67,7 @@ struct module to describe the types of columns in a table: @hrow{Format, C Type, Python type, Notes} -@row{\c x, N/A, N/A, N/A, pad byte\, no associated value} +@row{\c x, N/A, N/A, pad byte\, no associated value} @row{\c b, \c int8_t, \c int, signed byte} @row{\c B, \c uint8_t, \c int, unsigned byte} @row{\c h, \c int16_t, \c int, signed 16-bit} diff --git a/src/third_party/wiredtiger/src/docs/transactions.dox b/src/third_party/wiredtiger/src/docs/transactions.dox index c594b6b15c9..3c1233b0400 100644 --- a/src/third_party/wiredtiger/src/docs/transactions.dox +++ b/src/third_party/wiredtiger/src/docs/transactions.dox @@ -246,6 +246,15 @@ For example, for the timestamp values prepare_timestamp=100, timestamp is rounded up to the new prepare timestamp as commit_timestamp=200. +Configuring roundup_timestamps=(prepared=true) also allows setting +the prepared timestamps smaller than or equal to the latest active read timestamps. +Use this feature carefully as it may break repeated read. For example, consider a +transaction with a read timestamp set to 30 and a key that has a value with +timestamp 20. Before a prepared transaction (in another thread), reading the key +returns the value. Now a prepared transaction elsewhere in the system sets a prepared +timestamp at 30. The reader, trying to do a repeat read of the key, gets return a +prepared conflict error instead of the value. + Configuring roundup_timestamps=(read=true) causes the read timestamp to be rounded up to the oldest timestamp, if the read timestamp is greater than the oldest timestamp no change will be made. diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index c5f4b2d013e..e946fa2e2eb 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -666,7 +666,7 @@ select adaptive pthread mutexes by specifying
LSM merge threads option change
The WT_SESSION::create \c lsm=(merge_threads) configuration option has been -replaced by the W::wiredtiger_open \c lsm_manager=(worker_thread_max) option. +replaced by the ::wiredtiger_open \c lsm_manager=(worker_thread_max) option. The new version specifies a set of LSM threads that are shared across all LSM trees in a database, the older configuration was per LSM table.
diff --git a/src/third_party/wiredtiger/src/history/hs_verify.c b/src/third_party/wiredtiger/src/history/hs_verify.c index 3b73a587922..986e89fdb2b 100644 --- a/src/third_party/wiredtiger/src/history/hs_verify.c +++ b/src/third_party/wiredtiger/src/history/hs_verify.c @@ -75,7 +75,7 @@ __hs_verify_id( F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); WT_ERR_PANIC(session, WT_PANIC, "the associated history store key %s was not found in the data store %s", - __wt_buf_set_printable(session, key.data, key.size, prev_key), + __wt_buf_set_printable(session, key.data, key.size, false, prev_key), session->dhandle->name); } @@ -181,7 +181,7 @@ __wt_hs_verify(WT_SESSION_IMPL *session) F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); WT_ERR_PANIC(session, WT_PANIC, "Unable to find btree id %" PRIu32 " in the metadata file for the associated key %s", - btree_id, __wt_buf_set_printable(session, key.data, key.size, buf)); + btree_id, __wt_buf_set_printable(session, key.data, key.size, false, buf)); } WT_ERR(__wt_open_cursor(session, uri_data, NULL, NULL, &ds_cursor)); F_SET(ds_cursor, WT_CURSOR_RAW_OK); diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 68b89d99b0e..1892edd09a3 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -54,9 +54,10 @@ extern const WT_CONFIG_ENTRY *__wt_test_config_match(const char *test_name) extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_buf_set_printable(WT_SESSION_IMPL *session, const void *p, size_t size, - WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + bool hexonly, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_buf_set_printable_format(WT_SESSION_IMPL *session, const void *buffer, - size_t size, const char *format, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + size_t size, const char *format, bool hexonly, WT_ITEM *buf) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_buf_set_size(WT_SESSION_IMPL *session, uint64_t size, bool exact, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_cell_type_string(uint8_t type) @@ -2134,6 +2135,8 @@ static inline int __wt_txn_idle_cache_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_txn_modify_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, + WT_UPDATE *upd, wt_timestamp_t *prev_tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_modify_page_delete(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_op_set_key(WT_SESSION_IMPL *session, const WT_ITEM *key) @@ -2147,8 +2150,6 @@ static inline int __wt_txn_read_upd_list_internal(WT_SESSION_IMPL *session, WT_C WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_search_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_UPDATE *upd, wt_timestamp_t *prev_tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, u_int modify_type, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp, diff --git a/src/third_party/wiredtiger/src/include/serial_inline.h b/src/third_party/wiredtiger/src/include/serial_inline.h index 8e6813f2866..4f6384732dc 100644 --- a/src/third_party/wiredtiger/src/include/serial_inline.h +++ b/src/third_party/wiredtiger/src/include/serial_inline.h @@ -242,7 +242,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page * Check if our update is still permitted. */ while (!__wt_atomic_cas_ptr(srch_upd, upd->next, upd)) { - if ((ret = __wt_txn_update_check(session, cbt, upd->next = *srch_upd, &prev_upd_ts)) != 0) { + if ((ret = __wt_txn_modify_check(session, cbt, upd->next = *srch_upd, &prev_upd_ts)) != 0) { /* Free unused memory on error. */ __wt_free(session, upd); return (ret); diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 893179db355..e590bae7f20 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -743,6 +743,8 @@ struct __wt_connection_stats { int64_t txn_rts_pages_visited; int64_t txn_rts_hs_restore_tombstones; int64_t txn_rts_hs_restore_updates; + int64_t txn_rts_delete_rle_skipped; + int64_t txn_rts_stable_rle_skipped; int64_t txn_rts_sweep_hs_keys; int64_t txn_rts_tree_walk_skip_pages; int64_t txn_rts_upd_aborted; @@ -1014,6 +1016,8 @@ struct __wt_dsrc_stats { int64_t txn_rts_keys_restored; int64_t txn_rts_hs_restore_tombstones; int64_t txn_rts_hs_restore_updates; + int64_t txn_rts_delete_rle_skipped; + int64_t txn_rts_stable_rle_skipped; int64_t txn_rts_sweep_hs_keys; int64_t txn_rts_hs_removed; int64_t txn_checkpoint_obsolete_applied; diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index df81b29fe23..6d3b4e8fa55 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -1304,11 +1304,11 @@ __wt_txn_search_check(WT_SESSION_IMPL *session) } /* - * __wt_txn_update_check -- - * Check if the current transaction can update an item. + * __wt_txn_modify_check -- + * Check if the current transaction can modify an item. */ static inline int -__wt_txn_update_check( +__wt_txn_modify_check( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, wt_timestamp_t *prev_tsp) { WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 2da0d297fb1..e292d1c74fd 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -902,10 +902,12 @@ struct __wt_session { * @config{dump, configure the cursor for dump format inputs and outputs: "hex" selects a * simple hexadecimal format\, "json" selects a JSON format with each record formatted as * fields named by column names if available\, "pretty" selects a human-readable format - * (making it incompatible with the "load") and "print" selects a format where only - * non-printing characters are hexadecimal encoded. These formats are compatible with the - * @ref util_dump and @ref util_load commands., a string\, chosen from the following - * options: \c "hex"\, \c "json"\, \c "pretty"\, \c "print"; default empty.} + * (making it incompatible with the "load")\, "pretty_hex" is similar to "pretty" (also + * incompatible with "load") except raw byte data elements will be printed like "hex" + * format\, and "print" selects a format where only non-printing characters are hexadecimal + * encoded. These formats are compatible with the @ref util_dump and @ref util_load + * commands., a string\, chosen from the following options: \c "hex"\, \c "json"\, \c + * "pretty"\, \c "pretty_hex"\, \c "print"; default empty.} * @config{incremental = (, configure the cursor for block incremental backup usage. These * formats are only compatible with the backup data source; see @ref backup., a set of * related configuration options defined below.} @@ -1675,11 +1677,12 @@ struct __wt_session { * and the commit timestamp of this transaction can be rounded up. If the prepare timestamp * is less than the oldest timestamp\, the prepare timestamp will be rounded to the oldest * timestamp. If the commit timestamp is less than the prepare timestamp\, the commit - * timestamp will be rounded up to the prepare timestamp., a boolean flag; default \c - * false.} - * @config{    read, if the read timestamp is less than the - * oldest timestamp\, the read timestamp will be rounded up to the oldest timestamp., a - * boolean flag; default \c false.} + * timestamp will be rounded up to the prepare timestamp. Allows setting the prepared + * timestamp smaller than or equal to the latest active read timestamp., a boolean flag; + * default \c false.} + * @config{    read, if the read timestamp is less + * than the oldest timestamp\, the read timestamp will be rounded up to the oldest + * timestamp., a boolean flag; default \c false.} * @config{ ),,} * @config{sync, whether to sync log records when the transaction commits\, inherited from * ::wiredtiger_open \c transaction_sync., a boolean flag; default empty.} @@ -6008,140 +6011,144 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1428 /*! transaction: rollback to stable restored updates from history store */ #define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1429 +/*! transaction: rollback to stable skipping delete rle */ +#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1430 +/*! transaction: rollback to stable skipping stable rle */ +#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1431 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1430 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1432 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1431 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1433 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1432 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1434 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1433 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1435 /*! transaction: sessions scanned in each walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1434 +#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1436 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1435 +#define WT_STAT_CONN_TXN_SET_TS 1437 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1436 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1438 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1437 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1439 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1438 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1440 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1439 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1441 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1440 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1442 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1441 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1443 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1442 +#define WT_STAT_CONN_TXN_BEGIN 1444 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1443 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1445 /*! * transaction: transaction checkpoint currently running for history * store file */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1444 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1446 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1445 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1447 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1446 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1448 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1447 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1449 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1448 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1450 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1449 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1451 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1450 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1452 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1451 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1453 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1452 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1454 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1453 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1455 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1454 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1456 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1455 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1457 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1456 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1458 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1457 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1459 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1458 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1460 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1461 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1462 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1461 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1463 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1462 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1464 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1463 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1465 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1464 +#define WT_STAT_CONN_TXN_CHECKPOINT 1466 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1465 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1467 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1466 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1468 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1467 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1469 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1468 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1470 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1469 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1471 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1470 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1472 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1471 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1473 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1472 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1474 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1473 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1475 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1474 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1476 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1475 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1477 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1476 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1478 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1477 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1479 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1478 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1480 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1479 +#define WT_STAT_CONN_TXN_COMMIT 1481 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1480 +#define WT_STAT_CONN_TXN_ROLLBACK 1482 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1481 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1483 /*! * @} @@ -6772,14 +6779,18 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2210 /*! transaction: rollback to stable restored updates from history store */ #define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2211 +/*! transaction: rollback to stable skipping delete rle */ +#define WT_STAT_DSRC_TXN_RTS_DELETE_RLE_SKIPPED 2212 +/*! transaction: rollback to stable skipping stable rle */ +#define WT_STAT_DSRC_TXN_RTS_STABLE_RLE_SKIPPED 2213 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2212 +#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2214 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2213 +#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2215 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2214 +#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2216 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2215 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2217 /*! * @} diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 9518a3b63c3..4ea57b8acc7 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -327,7 +327,9 @@ __wt_rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) WT_RET(__wt_rec_split_init(session, r, page, pageref->ref_recno, btree->maxleafpage)); /* Copy the original, disk-image bytes into place. */ - memcpy(r->first_free, page->pg_fix_bitf, __bitstr_size((size_t)page->entries * btree->bitcnt)); + if (page->entries != 0) + memcpy( + r->first_free, page->pg_fix_bitf, __bitstr_size((size_t)page->entries * btree->bitcnt)); /* Update any changes to the original on-page data items. */ WT_SKIP_FOREACH (ins, WT_COL_UPDATE_SINGLE(page)) { diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 667e8c119b7..9ad305792b9 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -2058,7 +2058,7 @@ __rec_split_dump_keys(WT_SESSION_IMPL *session, WT_RECONCILE *r) for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) __wt_verbose(session, WT_VERB_SPLIT, "starting key %s", __wt_buf_set_printable( - session, WT_IKEY_DATA(multi->key.ikey), multi->key.ikey->size, tkey)); + session, WT_IKEY_DATA(multi->key.ikey), multi->key.ikey->size, false, tkey)); __wt_scr_free(session, &tkey); } else for (multi = r->multi, i = 0; i < r->multi_next; ++multi, ++i) diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 33c80a3a0ed..a6fff0b3e6c 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -117,9 +117,17 @@ err: * Set the contents of the buffer to a printable representation of a byte string. */ const char * -__wt_buf_set_printable(WT_SESSION_IMPL *session, const void *p, size_t size, WT_ITEM *buf) +__wt_buf_set_printable( + WT_SESSION_IMPL *session, const void *p, size_t size, bool hexonly, WT_ITEM *buf) { - if (__wt_raw_to_esc_hex(session, p, size, buf)) { + WT_DECL_RET; + + if (hexonly) + ret = __wt_raw_to_hex(session, p, size, buf); + else + ret = __wt_raw_to_esc_hex(session, p, size, buf); + + if (ret != 0) { buf->data = "[Error]"; buf->size = strlen("[Error]"); } @@ -132,8 +140,8 @@ __wt_buf_set_printable(WT_SESSION_IMPL *session, const void *p, size_t size, WT_ * format. */ const char * -__wt_buf_set_printable_format( - WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *format, WT_ITEM *buf) +__wt_buf_set_printable_format(WT_SESSION_IMPL *session, const void *buffer, size_t size, + const char *format, bool hexonly, WT_ITEM *buf) { WT_DECL_ITEM(tmp); WT_DECL_PACK_VALUE(pv); @@ -166,7 +174,7 @@ __wt_buf_set_printable_format( if (tmp == NULL) WT_ERR(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_buf_catfmt(session, buf, "%s%s", sep, - __wt_buf_set_printable(session, pv.u.item.data, pv.u.item.size, tmp))); + __wt_buf_set_printable(session, pv.u.item.data, pv.u.item.size, hexonly, tmp))); break; case 'b': case 'h': @@ -203,7 +211,7 @@ err: * is truncated, and then passed here by a page debugging routine). Our current callers aren't * interested in error handling in such cases, return a byte string instead. */ - return (__wt_buf_set_printable(session, buffer, size, buf)); + return (__wt_buf_set_printable(session, buffer, size, hexonly, buf)); } /* diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index bc13dd8aab7..c33d3b60bef 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -221,6 +221,8 @@ static const char *const __stats_dsrc_desc[] = { "transaction: rollback to stable keys restored", "transaction: rollback to stable restored tombstones from history store", "transaction: rollback to stable restored updates from history store", + "transaction: rollback to stable skipping delete rle", + "transaction: rollback to stable skipping stable rle", "transaction: rollback to stable sweeping history store keys", "transaction: rollback to stable updates removed from history store", "transaction: transaction checkpoints due to obsolete pages", @@ -477,6 +479,8 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->txn_rts_keys_restored = 0; stats->txn_rts_hs_restore_tombstones = 0; stats->txn_rts_hs_restore_updates = 0; + stats->txn_rts_delete_rle_skipped = 0; + stats->txn_rts_stable_rle_skipped = 0; stats->txn_rts_sweep_hs_keys = 0; stats->txn_rts_hs_removed = 0; stats->txn_checkpoint_obsolete_applied = 0; @@ -720,6 +724,8 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->txn_rts_keys_restored += from->txn_rts_keys_restored; to->txn_rts_hs_restore_tombstones += from->txn_rts_hs_restore_tombstones; to->txn_rts_hs_restore_updates += from->txn_rts_hs_restore_updates; + to->txn_rts_delete_rle_skipped += from->txn_rts_delete_rle_skipped; + to->txn_rts_stable_rle_skipped += from->txn_rts_stable_rle_skipped; to->txn_rts_sweep_hs_keys += from->txn_rts_sweep_hs_keys; to->txn_rts_hs_removed += from->txn_rts_hs_removed; to->txn_checkpoint_obsolete_applied += from->txn_checkpoint_obsolete_applied; @@ -971,6 +977,8 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->txn_rts_keys_restored += WT_STAT_READ(from, txn_rts_keys_restored); to->txn_rts_hs_restore_tombstones += WT_STAT_READ(from, txn_rts_hs_restore_tombstones); to->txn_rts_hs_restore_updates += WT_STAT_READ(from, txn_rts_hs_restore_updates); + to->txn_rts_delete_rle_skipped += WT_STAT_READ(from, txn_rts_delete_rle_skipped); + to->txn_rts_stable_rle_skipped += WT_STAT_READ(from, txn_rts_stable_rle_skipped); to->txn_rts_sweep_hs_keys += WT_STAT_READ(from, txn_rts_sweep_hs_keys); to->txn_rts_hs_removed += WT_STAT_READ(from, txn_rts_hs_removed); to->txn_checkpoint_obsolete_applied += WT_STAT_READ(from, txn_checkpoint_obsolete_applied); @@ -1419,6 +1427,8 @@ static const char *const __stats_connection_desc[] = { "transaction: rollback to stable pages visited", "transaction: rollback to stable restored tombstones from history store", "transaction: rollback to stable restored updates from history store", + "transaction: rollback to stable skipping delete rle", + "transaction: rollback to stable skipping stable rle", "transaction: rollback to stable sweeping history store keys", "transaction: rollback to stable tree walk skipping pages", "transaction: rollback to stable updates aborted", @@ -1942,6 +1952,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->txn_rts_pages_visited = 0; stats->txn_rts_hs_restore_tombstones = 0; stats->txn_rts_hs_restore_updates = 0; + stats->txn_rts_delete_rle_skipped = 0; + stats->txn_rts_stable_rle_skipped = 0; stats->txn_rts_sweep_hs_keys = 0; stats->txn_rts_tree_walk_skip_pages = 0; stats->txn_rts_upd_aborted = 0; @@ -2476,6 +2488,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->txn_rts_pages_visited += WT_STAT_READ(from, txn_rts_pages_visited); to->txn_rts_hs_restore_tombstones += WT_STAT_READ(from, txn_rts_hs_restore_tombstones); to->txn_rts_hs_restore_updates += WT_STAT_READ(from, txn_rts_hs_restore_updates); + to->txn_rts_delete_rle_skipped += WT_STAT_READ(from, txn_rts_delete_rle_skipped); + to->txn_rts_stable_rle_skipped += WT_STAT_READ(from, txn_rts_stable_rle_skipped); to->txn_rts_sweep_hs_keys += WT_STAT_READ(from, txn_rts_sweep_hs_keys); to->txn_rts_tree_walk_skip_pages += WT_STAT_READ(from, txn_rts_tree_walk_skip_pages); to->txn_rts_upd_aborted += WT_STAT_READ(from, txn_rts_upd_aborted); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_opener.c b/src/third_party/wiredtiger/src/tiered/tiered_opener.c index 0e28bcdcbcb..e65d52e4694 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_opener.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_opener.c @@ -35,15 +35,17 @@ __tiered_opener_open(WT_BLOCK_FILE_OPENER *opener, WT_SESSION_IMPL *session, uin object_name = tiered->tiers[WT_TIERED_INDEX_LOCAL].name; if (!WT_PREFIX_SKIP(object_name, "file:")) WT_RET_MSG(session, EINVAL, "expected a 'file:' URI"); + WT_ERR(__wt_open(session, object_name, type, flags, fhp)); } else { WT_ERR( __wt_tiered_name(session, &tiered->iface, object_id, WT_TIERED_NAME_OBJECT, &object_uri)); object_name = object_uri; WT_PREFIX_SKIP_REQUIRED(session, object_name, "object:"); bstorage = tiered->bstorage; + flags |= WT_FS_OPEN_READONLY; + WT_WITH_BUCKET_STORAGE( + bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); } - WT_WITH_BUCKET_STORAGE( - bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); err: __wt_free(session, object_uri); return (ret); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 68e0e45a8ea..35eb2e08d6f 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -723,12 +723,14 @@ __wt_txn_release(WT_SESSION_IMPL *session) } /* - * __txn_append_hs_record -- - * Append the update older than the prepared update to the update chain + * __txn_locate_hs_record -- + * Locate the update older than the prepared update in the history store and append it to the + * update chain if necessary. */ static int -__txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_PAGE *page, - WT_UPDATE *chain, bool commit, WT_UPDATE **fix_updp, bool *upd_appended) +__txn_locate_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_PAGE *page, + WT_UPDATE *chain, bool commit, WT_UPDATE **fix_updp, bool *upd_appended, + bool first_committed_upd_in_hs) { WT_DECL_ITEM(hs_value); WT_DECL_RET; @@ -769,10 +771,10 @@ __txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_PAGE * *fix_updp = upd; /* - * When the prepared update is getting committed, use the retrieved history store update to form - * a proper stop timestamp in the history store. + * When the prepared update is getting committed or the history store update is still on the + * update chain, no need to append it onto the update chain. */ - if (commit) + if (commit || first_committed_upd_in_hs) goto done; /* @@ -1079,14 +1081,14 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_DECL_RET; WT_ITEM hs_recno_key; WT_TXN *txn; - WT_UPDATE *fix_upd, *tombstone, *upd; + WT_UPDATE *first_committed_upd, *fix_upd, *tombstone, *upd; #ifdef HAVE_DIAGNOSTIC WT_UPDATE *head_upd; #endif size_t not_used; uint8_t *p, hs_recno_key_buf[WT_INTPACK64_MAXSIZE]; char ts_string[3][WT_TS_INT_STRING_SIZE]; - bool upd_appended; + bool first_committed_upd_in_hs, prepare_on_disk, upd_appended; hs_cursor = NULL; txn = session->txn; @@ -1125,9 +1127,23 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, if (upd == NULL || upd->prepare_state != WT_PREPARE_INPROGRESS) goto prepare_verify; - WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); + /* A prepared operation that is rolled back will not have a timestamp worth asserting on. */ + if (commit) + WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); + + for (first_committed_upd = upd; first_committed_upd != NULL && + (first_committed_upd->txnid == WT_TXN_ABORTED || + first_committed_upd->prepare_state == WT_PREPARE_INPROGRESS); + first_committed_upd = first_committed_upd->next) + ; + /* - * Retrieve the previous update from the history store and append it to the update chain. + * Locate the previous update from the history store and append it to the update chain if + * required. We know there may be content in the history store if the prepared update is written + * to the disk image or first committed update older than the prepared update is marked as + * WT_UPDATE_HS. The second case is rare but can happen if the eviction that writes the prepared + * update to the disk image fails after it has inserted the other updates of the key into the + * history store. * * We need to do this before we resolve the prepared updates because if we abort the prepared * updates first, the history search logic may race with other sessions modifying the same key @@ -1137,10 +1153,13 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * the update is also from the same prepared transaction, restore the update from history store * or remove the key. */ - if (F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS) && + prepare_on_disk = F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS) && (upd->type != WT_UPDATE_TOMBSTONE || (!commit && upd->next != NULL && upd->durable_ts == upd->next->durable_ts && - upd->txnid == upd->next->txnid && upd->start_ts == upd->next->start_ts))) { + upd->txnid == upd->next->txnid && upd->start_ts == upd->next->start_ts)); + first_committed_upd_in_hs = + first_committed_upd != NULL && F_ISSET(first_committed_upd, WT_UPDATE_HS); + if (prepare_on_disk || first_committed_upd_in_hs) { btree = S2BT(session); cbt = (WT_CURSOR_BTREE *)(*cursorp); @@ -1160,6 +1179,9 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, hs_cursor->set_key(hs_cursor, 4, btree->id, &hs_recno_key, WT_TS_MAX, UINT64_MAX); } WT_ERR_NOTFOUND_OK(__wt_curhs_search_near_before(session, hs_cursor), true); + + /* We should only get not found if the prepared update is on disk. */ + WT_ASSERT(session, ret != WT_NOTFOUND || prepare_on_disk); if (ret == WT_NOTFOUND && !commit) { /* * Allocate a tombstone and prepend it to the row so when we reconcile the update chain @@ -1182,8 +1204,8 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, WT_ERR(ret); tombstone = NULL; } else if (ret == 0) - WT_ERR(__txn_append_hs_record( - session, hs_cursor, cbt->ref->page, upd, commit, &fix_upd, &upd_appended)); + WT_ERR(__txn_locate_hs_record(session, hs_cursor, cbt->ref->page, upd, commit, &fix_upd, + &upd_appended, first_committed_upd)); else ret = 0; } @@ -1250,8 +1272,12 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * prepared updates are written to the data store. When the page is read back into memory, there * will be only one uncommitted prepared update. */ - if (fix_upd != NULL) + if (fix_upd != NULL) { WT_ERR(__txn_fixup_prepared_update(session, hs_cursor, fix_upd, commit)); + /* Clear the WT_UPDATE_HS flag as we should have removed it from the history store. */ + if (first_committed_upd_in_hs && !commit) + F_CLR(first_committed_upd, WT_UPDATE_HS); + } prepare_verify: #ifdef HAVE_DIAGNOSTIC diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 5bc0f245497..3ec6bb95934 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -639,7 +639,7 @@ err: */ static int __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, WT_ROW *rip, - wt_timestamp_t rollback_timestamp, uint64_t recno) + wt_timestamp_t rollback_timestamp, uint64_t recno, bool *is_ondisk_stable) { WT_CELL *kcell; WT_CELL_UNPACK_KV *vpack, _vpack; @@ -654,6 +654,10 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W vpack = &_vpack; upd = NULL; + /* Initialize the on-disk stable version flag. */ + if (is_ondisk_stable != NULL) + *is_ondisk_stable = false; + /* * Assert an exclusive or for rip and cip such that either only a cip for a column store or a * rip for a row store are passed into the function. @@ -768,9 +772,12 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_COL *cip, W __wt_timestamp_to_string(vpack->tw.durable_stop_ts, ts_string[4]), vpack->tw.stop_txn, prepared ? "true" : "false"); } - } else + } else { /* Stable version according to the timestamp. */ + if (is_ondisk_stable != NULL) + *is_ondisk_stable = true; return (0); + } if (rip != NULL) WT_ERR(__rollback_row_modify(session, page, rip, upd)); @@ -821,9 +828,19 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r kcell = WT_COL_PTR(page, cip); __wt_cell_unpack_kv(session, page->dsk, kcell, &unpack); rle = __wt_cell_rle(&unpack); - for (j = 0; j < rle; j++) - WT_RET(__rollback_abort_ondisk_kv( - session, ref, cip, NULL, rollback_timestamp, recno + j)); + if (unpack.type != WT_CELL_DEL) { + for (j = 0; j < rle; j++) { + WT_RET(__rollback_abort_ondisk_kv(session, ref, cip, NULL, rollback_timestamp, + recno + j, &stable_update_found)); + /* Skip processing all RLE if the on-disk version is stable. */ + if (stable_update_found) { + if (rle > 1) + WT_STAT_CONN_DATA_INCR(session, txn_rts_stable_rle_skipped); + break; + } + } + } else + WT_STAT_CONN_DATA_INCR(session, txn_rts_delete_rle_skipped); recno += rle; } else { recno++; @@ -910,7 +927,8 @@ __rollback_abort_row_leaf(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t * If there is no stable update found in the update list, abort any on-disk value. */ if (!stable_update_found) - WT_ERR(__rollback_abort_ondisk_kv(session, ref, NULL, rip, rollback_timestamp, 0)); + WT_ERR( + __rollback_abort_ondisk_kv(session, ref, NULL, rip, rollback_timestamp, 0, NULL)); } /* Mark the page as dirty to reconcile the page. */ diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 9fc79a78d72..9faa37d0c57 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -714,7 +714,19 @@ __wt_txn_set_prepare_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t prepare_ WT_RET_MSG(session, EINVAL, "commit timestamp should not have been set before the prepare timestamp"); - WT_RET(__txn_assert_after_reads(session, "prepare", prepare_ts)); + /* + * Allows setting the prepared timestamp smaller than or equal to the latest active read + * timestamp. This feature is necessary to find the largest key in the table even if that + * key has been deleted. Set this flag cautiously as it breaks repeated reads. + */ +#ifdef HAVE_DIAGNOSTIC + if (!F_ISSET(txn, WT_TXN_TS_ROUND_PREPARED)) + WT_RET(__txn_assert_after_reads(session, "prepare", prepare_ts)); + else + WT_RET(__wt_msg(session, + "Skip checking prepare timestamp %s against the latest active read timestamp.", + __wt_timestamp_to_string(prepare_ts, ts_string[0]))); +#endif /* * Check whether the prepare timestamp is less than the oldest timestamp. diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index 565660c5b66..727cf132ffc 100755 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -13,12 +13,13 @@ #define STRING_MATCH_CONFIG(s, item) \ (strncmp(s, (item).str, (item).len) == 0 && (s)[(item).len] == '\0') -static int dump_config(WT_SESSION *, const char *, WT_CURSOR *, bool, bool); +static int dump_config(WT_SESSION *, const char *, WT_CURSOR *, bool, bool, bool); static int dump_json_begin(WT_SESSION *); static int dump_json_end(WT_SESSION *); static int dump_json_separator(WT_SESSION *); static int dump_json_table_end(WT_SESSION *); -static int dump_prefix(WT_SESSION *, bool, bool); +static const char *get_dump_type(bool, bool, bool); +static int dump_prefix(WT_SESSION *, bool, bool, bool); static int dump_record(WT_CURSOR *, bool, bool); static int dump_suffix(WT_SESSION *, bool); static int dump_table_config(WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, bool); @@ -33,12 +34,16 @@ usage(void) static const char *options[] = {"-c checkpoint", "dump as of the named checkpoint (the default is the most recent version of the data)", "-f output", "dump to the specified file (the default is stdout)", "-j", - "dump in JSON format", "-p", "dump in human readable format (pretty-print)", "-r", - "dump in reverse order", "-t timestamp", + "dump in JSON format", "-p", + "dump in human readable format (pretty-print). The -p flag can be combined with -x. In this " + "case, raw data elements will be formatted like -x with hexadecimal encoding.", + "-r", "dump in reverse order", "-t timestamp", "dump as of the specified timestamp (the default is the most recent version of the data)", "-x", "dump all characters in a hexadecimal encoding (by default printable characters are not " - "encoded)", + "encoded). The -x flag can be combined with -p. In this case, the dump will be formatted " + "similar to -p except for raw data elements, which will look like -x with hexadecimal " + "encoding.", NULL, NULL}; util_usage( @@ -108,8 +113,13 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) ++format_specifiers; if (hex) ++format_specifiers; - if (format_specifiers > 1) { - fprintf(stderr, "%s: the -j, -p and -x dump options are incompatible\n", progname); + + /* Supported options are -j, -p, -x and a combination of -p and -x. */ + if (format_specifiers > 1 && !(pretty && hex)) { + fprintf(stderr, + "%s: the only possible options are -j, -p, -x and a combination of -p and -x. Other " + "options are incompatible\n", + progname); return (usage()); } @@ -119,7 +129,7 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) else if ((fp = fopen(ofile, "w")) == NULL) return (util_err(session, errno, "%s: open", ofile)); - if (json && (dump_json_begin(session) != 0 || dump_prefix(session, hex, json) != 0)) + if (json && (dump_json_begin(session) != 0 || dump_prefix(session, pretty, hex, json) != 0)) goto err; WT_RET(__wt_scr_alloc(session_impl, 0, &tmp)); @@ -147,8 +157,7 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) WT_ERR(__wt_buf_set(session_impl, tmp, "", 0)); if (checkpoint != NULL) WT_ERR(__wt_buf_catfmt(session_impl, tmp, "checkpoint=%s,", checkpoint)); - WT_ERR(__wt_buf_catfmt(session_impl, tmp, "dump=%s", - json ? "json" : (hex ? "hex" : (pretty ? "pretty" : "print")))); + WT_ERR(__wt_buf_catfmt(session_impl, tmp, "dump=%s", get_dump_type(pretty, hex, json))); if ((ret = session->open_cursor(session, uri, NULL, (char *)tmp->data, &cursor)) != 0) { fprintf(stderr, "%s: cursor open(%s) failed: %s\n", progname, uri, session->strerror(session, ret)); @@ -172,7 +181,7 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) /* Set the "ignore tombstone" flag on the underlying cursor. */ F_SET(hs_dump_cursor->child, WT_CURSTD_IGNORE_TOMBSTONE); } - if (dump_config(session, simpleuri, cursor, hex, json) != 0) + if (dump_config(session, simpleuri, cursor, pretty, hex, json) != 0) goto err; if (dump_record(cursor, reverse, json) != 0) @@ -241,7 +250,8 @@ time_pair_to_timestamp(WT_SESSION_IMPL *session_impl, char *ts_string, WT_ITEM * * Dump the config for the uri. */ static int -dump_config(WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool hex, bool json) +dump_config( + WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool pretty, bool hex, bool json) { WT_CURSOR *mcursor; WT_DECL_RET; @@ -260,7 +270,7 @@ dump_config(WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool hex, b */ mcursor->set_key(mcursor, uri); if ((ret = mcursor->search(mcursor)) == 0) { - if ((!json && dump_prefix(session, hex, json) != 0) || + if ((!json && dump_prefix(session, pretty, hex, json) != 0) || dump_table_config(session, mcursor, cursor, uri, json) != 0 || dump_suffix(session, json) != 0) ret = 1; @@ -559,12 +569,36 @@ match: return (util_cerr(cursor, "next", ret)); } +/* + * Returns dump type string based on the passed format flags + */ +static const char * +get_dump_type(bool pretty, bool hex, bool json) +{ + const char *result; + + result = NULL; + + if (json) + result = "json"; + else if (hex && pretty) + result = "pretty_hex"; + else if (hex) + result = "hex"; + else if (pretty) + result = "pretty"; + else + result = "print"; + + return result; +} + /* * dump_prefix -- * Output the dump file header prefix. */ static int -dump_prefix(WT_SESSION *session, bool hex, bool json) +dump_prefix(WT_SESSION *session, bool pretty, bool hex, bool json) { int vmajor, vminor, vpatch; @@ -577,7 +611,8 @@ dump_prefix(WT_SESSION *session, bool hex, bool json) if (!json && (fprintf(fp, "WiredTiger Dump (WiredTiger Version %d.%d.%d)\n", vmajor, vminor, vpatch) < 0 || - fprintf(fp, "Format=%s\n", hex ? "hex" : "print") < 0 || fprintf(fp, "Header\n") < 0)) + fprintf(fp, "Format=%s\n", (pretty && hex) ? "print hex" : hex ? "hex" : "print") < 0 || + fprintf(fp, "Header\n") < 0)) return (util_err(session, EIO, NULL)); return (0); diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 3bb80e94156..b7e7a9a952b 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -82,7 +82,6 @@ clock_thread(void *arg) testutil_check(g.conn->open_session(g.conn, NULL, NULL, &wt_session)); session = (WT_SESSION_IMPL *)wt_session; - g.ts_stable = 0; while (g.running) { __wt_writelock(session, &g.clock_lock); ++g.ts_stable; diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 83f1b1f6cef..962b1893305 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -5,22 +5,34 @@ set -e # Bypass this test for valgrind test "$TESTUTIL_BYPASS_VALGRIND" = "1" && exit 0 -# Temporarily disabled # Smoke-test checkpoints as part of running "make check". -#echo "checkpoint: 3 mixed tables" -#$TEST_WRAPPER ./t -T 3 -t m -# Temporarily disabled -#echo "checkpoint: 6 column-store tables" -#$TEST_WRAPPER ./t -T 6 -t c +echo "checkpoint: 3 mixed tables" +$TEST_WRAPPER ./t -T 3 -t m -# Temporarily disabled -#echo "checkpoint: 6 LSM tables" -#$TEST_WRAPPER ./t -T 6 -t l +echo "checkpoint: 6 column-store tables" +$TEST_WRAPPER ./t -T 6 -t c -# Temporarily disabled -#echo "checkpoint: 6 mixed tables" -#$TEST_WRAPPER ./t -T 6 -t m +echo "checkpoint: 6 column-store tables, named checkpoint" +$TEST_WRAPPER ./t -c 'TeSt' -T 6 -t c + +echo "checkpoint: 6 column-store tables with prepare" +$TEST_WRAPPER ./t -T 6 -t c -p + +echo "checkpoint: 6 column-store tables, named checkpoint with prepare" +$TEST_WRAPPER ./t -c 'TeSt' -T 6 -t c -p + +echo "checkpoint: column-store tables, stress history store. Sweep and timestamps" +$TEST_WRAPPER ./t -t c -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB + +echo "checkpoint: column-store tables, Sweep and timestamps" +$TEST_WRAPPER ./t -t c -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB + +echo "checkpoint: 6 LSM tables" +$TEST_WRAPPER ./t -T 6 -t l + +echo "checkpoint: 6 mixed tables" +$TEST_WRAPPER ./t -T 6 -t m echo "checkpoint: 6 row-store tables" $TEST_WRAPPER ./t -T 6 -t r @@ -34,18 +46,14 @@ $TEST_WRAPPER ./t -T 6 -t r -p echo "checkpoint: 6 row-store tables, named checkpoint with prepare" $TEST_WRAPPER ./t -c 'TeSt' -T 6 -t r -p -# Temporarily disabled -#echo "checkpoint: row-store tables, stress history store. Sweep and timestamps" -#$TEST_WRAPPER ./t -t r -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB +echo "checkpoint: row-store tables, stress history store. Sweep and timestamps" +$TEST_WRAPPER ./t -t r -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB -# Temporarily disabled -#echo "checkpoint: row-store tables, Sweep and timestamps" -#$TEST_WRAPPER ./t -t r -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB +echo "checkpoint: row-store tables, Sweep and timestamps" +$TEST_WRAPPER ./t -t r -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB -# Temporarily disabled -#echo "checkpoint: 3 mixed tables, with sweep" -#$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -s -n 100000 -k 100000 +echo "checkpoint: 3 mixed tables, with sweep" +$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -s -n 100000 -k 100000 -# Temporarily disabled -#echo "checkpoint: 3 mixed tables, with timestamps" -#$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -x -n 100000 -k 100000 +echo "checkpoint: 3 mixed tables, with timestamps" +$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -x -n 100000 -k 100000 diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index b236cb14308..90378cc0de9 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -140,6 +140,8 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(g.home, 512, working_dir); + g.ts_stable = 0; + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { cleanup(cnt == 1); /* Clean up previous runs */ @@ -171,7 +173,7 @@ main(int argc, char *argv[]) free(g.cookies); g.cookies = NULL; if ((ret = wt_shutdown()) != 0) { - (void)log_print_err("Start workers failed", ret, 1); + (void)log_print_err("Shutdown failed", ret, 1); break; } } @@ -250,7 +252,6 @@ cleanup(bool remove_dir) g.running = 0; g.ntables_created = 0; g.ts_oldest = 0; - g.ts_stable = 0; if (remove_dir) testutil_make_work_dir(g.home); diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index 2cd947fbd79..05b9a83b75b 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -280,11 +280,12 @@ real_worker(void) } else testutil_check(__wt_snprintf( buf, sizeof(buf), "commit_timestamp=%x", g.ts_stable + 1)); - __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); if ((ret = session->commit_transaction(session, buf)) != 0) { + __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); (void)log_print_err("real_worker:commit_transaction", ret, 1); goto err; } + __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); start_txn = true; /* Occasionally reopen cursors after committing. */ if (next_rnd % 13 == 0) { diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt b/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt index f5b5793491d..99707d70d08 100644 --- a/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt +++ b/src/third_party/wiredtiger/test/cppsuite/configs/hs_cleanup_stress.txt @@ -26,9 +26,9 @@ runtime_monitor= enabled=true, limit=1900000000, ), - # Seems to insert around 490K records. Give it +-25K margin. + # Seems to insert around 477K records. Give it +-20K margin. # Seems to remove 180K records. Give it a similar margin. - postrun_statistics=[cache_hs_insert:470000:515000, cc_pages_removed:170000:200000] + postrun_statistics=[cache_hs_insert:457000:497000, cc_pages_removed:170000:200000] ), timestamp_manager= ( diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx index e49590da7d7..4ce0de2e381 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test.cxx @@ -137,8 +137,8 @@ test::run() /* The test will run for the duration as defined in the config. */ duration_seconds = _config->get_int(DURATION_SECONDS); testutil_assert(duration_seconds >= 0); - logger::log_msg( - LOG_INFO, "Waiting {" + std::to_string(duration_seconds) + "} for testing to complete."); + logger::log_msg(LOG_INFO, + "Waiting {" + std::to_string(duration_seconds) + "} seconds for testing to complete."); std::this_thread::sleep_for(std::chrono::seconds(duration_seconds)); /* End the test by calling finish on all known components. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx index 7cc113f8ad8..3b7d2d312aa 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/timestamp_manager.cxx @@ -69,7 +69,7 @@ timestamp_manager::load() void timestamp_manager::do_work() { - std::string config; + std::string config, log_msg; /* latest_ts_s represents the time component of the latest timestamp provided. */ wt_timestamp_t latest_ts_s; @@ -82,7 +82,7 @@ timestamp_manager::do_work() */ testutil_assert(latest_ts_s >= _stable_ts); if ((latest_ts_s - _stable_ts) > _stable_lag) { - logger::log_msg(LOG_INFO, "Timestamp_manager: Stable timestamp expired."); + log_msg = "Timestamp_manager: Stable timestamp expired."; _stable_ts = latest_ts_s; config += std::string(STABLE_TS) + "=" + decimal_to_hex(_stable_ts); } @@ -94,13 +94,19 @@ timestamp_manager::do_work() wt_timestamp_t new_oldest_ts = _oldest_ts; testutil_assert(_stable_ts >= _oldest_ts); if ((_stable_ts - _oldest_ts) > _oldest_lag) { - logger::log_msg(LOG_INFO, "Timestamp_manager: Oldest timestamp expired."); + if (log_msg.empty()) + log_msg = "Timestamp_manager: Oldest timestamp expired."; + else + log_msg += " Oldest timestamp expired."; new_oldest_ts = _stable_ts - _oldest_lag; if (!config.empty()) config += ","; config += std::string(OLDEST_TS) + "=" + decimal_to_hex(new_oldest_ts); } + if (!log_msg.empty()) + logger::log_msg(LOG_INFO, log_msg); + /* * Save the new timestamps. Any timestamps that we're viewing from another thread should be set * AFTER we've saved the new timestamps to avoid races where we sweep data that is not yet diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx index e18125446fe..ebd73acf286 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/database_operation.cxx @@ -52,8 +52,9 @@ populate_worker(thread_context *tc) for (uint64_t i = 0; i < tc->key_count; ++i) { /* Start a txn. */ tc->transaction.begin(); - if (!tc->insert(cursor, coll.id, i)) { - /* We failed to insert, and our transaction was rolled back retry. */ + if (tc->insert(cursor, coll.id, i)) { + /* We failed to insert, rollback our transaction and retry. */ + tc->transaction.rollback(); --i; continue; } @@ -162,24 +163,31 @@ database_operation::insert_operation(thread_context *tc) auto &cc = ccv[counter]; while (tc->transaction.active() && tc->running()) { /* Insert a key value pair. */ - if (!tc->insert(cc.cursor, cc.coll.id, start_key + added_count)) { - committed = false; - break; + bool rollback_required = tc->insert(cc.cursor, cc.coll.id, start_key + added_count); + if (!rollback_required) { + added_count++; + if (tc->transaction.can_commit()) { + rollback_required = tc->transaction.commit(); + if (!rollback_required) + /* + * We need to inform the database model that we've added these keys as some + * other thread may rely on the key_count data. Only do so if we + * successfully committed. + */ + cc.coll.increase_key_count(added_count); + } } - added_count++; - tc->transaction.try_commit(); + + if (rollback_required) { + added_count = 0; + tc->transaction.rollback(); + } + /* Sleep the duration defined by the op_rate. */ tc->sleep(); } /* Reset our cursor to avoid pinning content. */ testutil_check(cc.cursor->reset(cc.cursor.get())); - - /* - * We need to inform the database model that we've added these keys as some other thread may - * rely on the key_count data. Only do so if we successfully committed. - */ - if (committed) - cc.coll.increase_key_count(added_count); counter++; if (counter == collections_per_thread) counter = 0; @@ -208,9 +216,17 @@ database_operation::read_operation(thread_context *tc) tc->transaction.begin(); while (tc->transaction.active() && tc->running()) { - if (tc->next(cursor) == WT_ROLLBACK) - /* We got an error, our transaction has been rolled back. */ - break; + auto ret = cursor->next(cursor.get()); + if (ret != 0) { + if (ret == WT_NOTFOUND) { + cursor->reset(cursor.get()); + } else if (ret == WT_ROLLBACK) { + tc->transaction.rollback(); + tc->sleep(); + continue; + } else + testutil_die(ret, "Unexpected error returned from cursor->next()"); + } tc->transaction.add_op(); tc->transaction.try_rollback(); tc->sleep(); @@ -263,21 +279,21 @@ database_operation::update_operation(thread_context *tc) /* Choose a random key to update. */ uint64_t key_id = random_generator::instance().generate_integer(0, coll.get_key_count() - 1); - bool successful_update = tc->update(cursor, coll.id, tc->key_to_string(key_id)); + bool rollback_required = tc->update(cursor, coll.id, tc->key_to_string(key_id)); /* Reset our cursor to avoid pinning content. */ testutil_check(cursor->reset(cursor.get())); - /* We received a rollback in update. */ - if (!successful_update) - continue; - /* Commit the current transaction if we're able to. */ - tc->transaction.try_commit(); + if (!rollback_required && tc->transaction.can_commit()) + rollback_required = tc->transaction.commit(); + + if (rollback_required) + tc->transaction.rollback(); } - /* Make sure the last operation is committed now the work is finished. */ + /* Make sure the last operation is rolled back now the work is finished. */ if (tc->transaction.active()) - tc->transaction.commit(); + tc->transaction.rollback(); } } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx index 321cc45b61b..aadf992133e 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.cxx @@ -71,6 +71,7 @@ transaction_context::begin(const std::string &config) random_generator::instance().generate_integer(_min_op_count, _max_op_count); _op_count = 0; _in_txn = true; + _needs_rollback = false; } void @@ -80,21 +81,22 @@ transaction_context::try_begin(const std::string &config) begin(config); } -void +/* It's possible to receive rollback in commit which is handled internally. */ +bool transaction_context::commit(const std::string &config) { + WT_DECL_RET; testutil_assert(_in_txn); - testutil_check( - _session->commit_transaction(_session, config.empty() ? nullptr : config.c_str())); - _op_count = 0; - _in_txn = false; -} - -void -transaction_context::try_commit(const std::string &config) -{ - if (can_commit_rollback()) - commit(config); + if ((ret = _session->commit_transaction(_session, config.empty() ? nullptr : config.c_str())) != + 0) { + logger::log_msg(LOG_WARN, + "Failed to commit transaction in commit, received error code: " + std::to_string(ret)); + _needs_rollback = true; + } else { + _op_count = 0; + _in_txn = false; + } + return (_needs_rollback); } void @@ -103,6 +105,7 @@ transaction_context::rollback(const std::string &config) testutil_assert(_in_txn); testutil_check( _session->rollback_transaction(_session, config.empty() ? nullptr : config.c_str())); + _needs_rollback = false; _op_count = 0; _in_txn = false; } @@ -110,7 +113,7 @@ transaction_context::rollback(const std::string &config) void transaction_context::try_rollback(const std::string &config) { - if (can_commit_rollback()) + if (can_rollback()) rollback(config); } @@ -124,8 +127,20 @@ transaction_context::set_commit_timestamp(wt_timestamp_t ts) testutil_check(_session->timestamp_transaction(_session, config.c_str())); } +void +transaction_context::set_needs_rollback(bool rollback) +{ + _needs_rollback = rollback; +} + bool -transaction_context::can_commit_rollback() +transaction_context::can_commit() +{ + return (!_needs_rollback && can_rollback()); +} + +bool +transaction_context::can_rollback() { return (_in_txn && _op_count >= _target_op_count); } @@ -184,8 +199,8 @@ thread_context::update(scoped_cursor &cursor, uint64_t collection_id, const std: ret = cursor->update(cursor.get()); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(); - return (false); + transaction.set_needs_rollback(true); + return (true); } else testutil_die(ret, "unhandled error while trying to update a key"); } @@ -193,14 +208,14 @@ thread_context::update(scoped_cursor &cursor, uint64_t collection_id, const std: tracking_operation::INSERT, collection_id, key.c_str(), value.c_str(), ts, op_track_cursor); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(); - return (false); + transaction.set_needs_rollback(true); + return (true); } else testutil_die( ret, "unhandled error while trying to save an update to the tracking table"); } transaction.add_op(); - return (true); + return (false); } bool @@ -226,8 +241,8 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k ret = cursor->insert(cursor.get()); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(); - return (false); + transaction.set_needs_rollback(true); + return (true); } else testutil_die(ret, "unhandled error while trying to insert a key"); } @@ -235,38 +250,14 @@ thread_context::insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t k tracking_operation::INSERT, collection_id, key.c_str(), value.c_str(), ts, op_track_cursor); if (ret != 0) { if (ret == WT_ROLLBACK) { - transaction.rollback(); - return (false); + transaction.set_needs_rollback(true); + return (true); } else testutil_die( ret, "unhandled error while trying to save an insert to the tracking table"); } transaction.add_op(); - return (true); -} - -int -thread_context::next(scoped_cursor &cursor) -{ - WT_DECL_RET; - - ret = cursor->next(cursor.get()); - - if (ret == WT_NOTFOUND) { - testutil_check(cursor->reset(cursor.get())); - return (ret); - } - - if (ret == WT_ROLLBACK) { - transaction.rollback(); - testutil_check(cursor->reset(cursor.get())); - return (ret); - } - - if (ret != 0) - testutil_die(ret, "cursor->next() failed with an unexpected error."); - - return (0); + return (false); } void diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 5255740f59c..7bb6ac66cb6 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -71,17 +71,28 @@ class transaction_context { void begin(const std::string &config = ""); /* Begin a transaction if we are not currently in one. */ void try_begin(const std::string &config = ""); - void commit(const std::string &config = ""); - /* Attempt to commit the transaction given the requirements are met. */ - void try_commit(const std::string &config = ""); + /* + * Commit a transaction and return true if a rollback is required. + */ + bool commit(const std::string &config = ""); + /* Rollback a transaction, failure will abort the test. */ void rollback(const std::string &config = ""); /* Attempt to rollback the transaction given the requirements are met. */ void try_rollback(const std::string &config = ""); /* Set a commit timestamp. */ void set_commit_timestamp(wt_timestamp_t ts); - - private: - bool can_commit_rollback(); + /* Set that the transaction needs to be rolled back. */ + void set_needs_rollback(bool rollback); + /* + * Returns true if a transaction can be committed as determined by the op count and the state of + * the transaction. + */ + bool can_commit(); + /* + * Returns true if a transaction can be rolled back as determined by the op count and the state + * of the transaction. + */ + bool can_rollback(); private: /* @@ -99,6 +110,7 @@ class transaction_context { int64_t _max_op_count = INT64_MAX; int64_t _target_op_count = 0; bool _in_txn = false; + bool _needs_rollback = false; WT_SESSION *_session = nullptr; timestamp_manager *_timestamp_manager = nullptr; @@ -124,25 +136,17 @@ class thread_context { /* * Generic update function, takes a collection_id and key, will generate the value. * - * Returns true if it successfully updates the key, false if it receives rollback from the API. + * Returns true if a rollback is required. */ bool update(scoped_cursor &cursor, uint64_t collection_id, const std::string &key); /* * Generic insert function, takes a collection_id and key_id, will generate the value. * - * Returns true if it successfully inserts the key, false if it receives rollback from the API. + * Returns true if a rollback is required. */ bool insert(scoped_cursor &cursor, uint64_t collection_id, uint64_t key_id); - /* - * Generic next function. - * - * Handles rollback and not found internally, but will return the error code to the caller so - * the caller can distinguish between them. - */ - int next(scoped_cursor &cursor); - void sleep(); bool running() const; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx index 4dd073cc115..03de6947a86 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/hs_cleanup.cxx @@ -62,8 +62,24 @@ class hs_cleanup : public test { while (tc->running()) { tc->sleep(); - if (tc->next(cursor) != 0) - continue; + auto ret = cursor->next(cursor.get()); + if (ret != 0) { + if (ret == WT_NOTFOUND) { + cursor->reset(cursor.get()); + continue; + } + if (ret == WT_ROLLBACK) { + /* + * As a result of the logic in this test its possible that the previous next + * call can happen outside the context of a transaction. Assert that we are in + * one if we got a rollback. + */ + testutil_check(tc->transaction.can_rollback()); + tc->transaction.rollback(); + continue; + } + testutil_die(ret, "Unexpected error returned from cursor->next()"); + } testutil_check(cursor->get_key(cursor.get(), &key_tmp)); @@ -75,14 +91,17 @@ class hs_cleanup : public test { * API doesn't guarantee our buffer will still be valid once it is called, as such we * copy the buffer and then pass it into the API. */ - if (!tc->update(cursor, coll.id, key_value_t(key_tmp))) - continue; + bool rollback_required = tc->update(cursor, coll.id, key_value_t(key_tmp)); /* Commit our transaction. */ - tc->transaction.try_commit(); + if (!rollback_required && tc->transaction.can_commit()) + rollback_required = tc->transaction.commit(); + + if (rollback_required) + tc->transaction.rollback(); } /* Ensure our last transaction is resolved. */ if (tc->transaction.active()) - tc->transaction.commit(); + tc->transaction.rollback(); } }; diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index c863f47cbd3..26789a1ac9b 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -162,15 +162,9 @@ thread_run(void *arg) testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); -#if 0 - /* - * Make sure that alternative threads operate on column-store table - * - * FIXME-WT-6125: temporarily turn off column store test. - */ + /* Make alternate threads operate on the column-store table. */ if (td->id % 2 != 0) columnar_table = true; -#endif if (columnar_table) testutil_check(session->open_cursor(session, col_uri, NULL, NULL, &cursor)); @@ -384,7 +378,6 @@ recover_and_verify(uint32_t nthreads) fatal = false; for (i = 0; i < nthreads; ++i) { -#if 0 /* * Every alternative thread is operated on column-store table. Make sure that proper cursor * is used for verification of recovered records. @@ -396,11 +389,6 @@ recover_and_verify(uint32_t nthreads) columnar_table = false; cursor = row_cursor; } -#else - /* FIXME-WT-6125: temporarily turn off column store test. */ - columnar_table = false; - cursor = row_cursor; -#endif middle = 0; testutil_check(__wt_snprintf(fname[DELETE_RECORD_FILE_ID], diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index ec9fcec8175..6fa41f0d82c 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -150,11 +150,13 @@ thread_ts_run(void *arg) THREAD_DATA *td; int dbg; char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE]; + bool first; td = (THREAD_DATA *)arg; __wt_random_init(&rnd); testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + first = true; /* Update the oldest timestamp every 1 millisecond. */ for (;;) { /* @@ -166,13 +168,15 @@ thread_ts_run(void *arg) testutil_check(pthread_rwlock_unlock(&ts_lock)); testutil_assert(ret == 0 || ret == WT_NOTFOUND); if (ret == 0) { - /* - * Set both the oldest and stable timestamp so that we don't need to maintain read - * availability at older timestamps. - */ - testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), - "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string)); + /* Periodically let the oldest timestamp lag. */ + if (!first && __wt_random(&rnd) % 4 == 0) + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "stable_timestamp=%s", ts_string)); + else + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string)); testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + first = false; /* * Set and reset the checkpoint retention setting on a regular basis. We want to test * racing with the internal archive thread while we're here. diff --git a/src/third_party/wiredtiger/test/cursor_order/Makefile.am b/src/third_party/wiredtiger/test/cursor_order/Makefile.am index f5330b57d3b..448f8f95772 100644 --- a/src/third_party/wiredtiger/test/cursor_order/Makefile.am +++ b/src/third_party/wiredtiger/test/cursor_order/Makefile.am @@ -9,7 +9,7 @@ cursor_order_LDADD = $(top_builddir)/test/utility/libtest_util.la cursor_order_LDADD +=$(top_builddir)/libwiredtiger.la cursor_order_LDFLAGS = -static -TESTS = $(noinst_PROGRAMS) +TESTS = smoke.sh clean-local: rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/test/cursor_order/smoke.sh b/src/third_party/wiredtiger/test/cursor_order/smoke.sh new file mode 100755 index 00000000000..13165c82d3d --- /dev/null +++ b/src/third_party/wiredtiger/test/cursor_order/smoke.sh @@ -0,0 +1,14 @@ +#! /bin/sh + +set -e + +# Smoke-test cursors as part of running "make check". + +echo "cursor_order: rows" +$TEST_WRAPPER ./cursor_order -tr + +echo "cursor_order: variable-length columns" +$TEST_WRAPPER ./cursor_order -tv + +echo "cursor_order: fixed-length columns" +$TEST_WRAPPER ./cursor_order -tf diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 5222df53fc0..5bc5fa6580b 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -116,7 +116,7 @@ functions: mkdir -p cmake_build cd cmake_build $CMAKE \ - ${posix_configure_flags|-DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_STATIC=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL} -G "${cmake_generator|Ninja}" ./.. + ${posix_configure_flags|-DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_STATIC=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL} -G "${cmake_generator|Ninja}" ./.. fi elif [ "$OS" != "Windows_NT" ]; then # Compiling with Autoconf/Libtool. @@ -2623,7 +2623,20 @@ tasks: virtualenv -p python3 venv source venv/bin/activate pip3 install lorem pymongo - ./run_many_coll.sh ../../mongo/build/install/bin/mongod mongodb.log config/many-collection-testing many-collection clean-and-populate + ./run_many_coll.sh ../../mongo/build/opt/install/bin/mongod mongodb.log config/many-collection-testing many-collection clean-and-populate + - command: shell.exec + params: + working_dir: mongo-tests/largescale + shell: bash + silent: true + script: | + set -o errexit + set -o verbose + virtualenv -p python3 venv + source venv/bin/activate + pip3 install "pymongo[srv]" + res_dir=`find ./ -type d -name "many-collection-[0-9]*" -print` + ./upload-results-atlas.py ${atlas_wt_perf_test_user} ${atlas_wt_perf_pass} wt-perf-tests many-collection-test ${branch_name} $res_dir/results/results.json - name: cyclomatic-complexity commands: @@ -2689,7 +2702,7 @@ buildvariants: - ubuntu1804-test expansions: test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(echo "`grep -c ^processor /proc/cpuinfo` * 2" | bc) cmake_generator: Ninja @@ -3069,7 +3082,7 @@ buildvariants: batchtime: 10080 # 7 days expansions: test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) cmake_generator: Ninja @@ -3108,7 +3121,7 @@ buildvariants: batchtime: 10080 # 7 days expansions: test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) - posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../build_cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_PYTHON=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL python_binary: '/opt/mongodbtoolchain/v3/bin/python3' smp_command: -j $(grep -c ^processor /proc/cpuinfo) cmake_generator: Ninja diff --git a/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 b/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 index 57639d8b776..001a93dbab4 100644 --- a/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 +++ b/src/third_party/wiredtiger/test/evergreen/build_windows.ps1 @@ -27,7 +27,7 @@ cd cmake_build # Configure build with CMake. if( $configure -eq $true) { - C:\cmake\bin\cmake --no-warn-unused-cli -DSWIG_DIR="C:\swigwin-3.0.2" -DSWIG_EXECUTABLE="C:\swigwin-3.0.2\swig.exe" -DCMAKE_BUILD_TYPE='None' -DENABLE_PYTHON=1 -DENABLE_STRICT=1 -DCMAKE_TOOLCHAIN_FILE='..\build_cmake\toolchains\cl.cmake' -G "Ninja" ..\. + C:\cmake\bin\cmake --no-warn-unused-cli -DSWIG_DIR="C:\swigwin-3.0.2" -DSWIG_EXECUTABLE="C:\swigwin-3.0.2\swig.exe" -DCMAKE_BUILD_TYPE='None' -DENABLE_PYTHON=1 -DENABLE_STRICT=1 -DCMAKE_TOOLCHAIN_FILE='..\cmake\toolchains\cl.cmake' -G "Ninja" ..\. } # Execute Ninja build. diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index a2fcc71c93e..461875e33bb 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -20,27 +20,27 @@ usage() { echo "usage: $0 [-aEFRSv] [-b format-binary] [-c config] [-e env-var]" echo " [-h home] [-j parallel-jobs] [-n total-jobs] [-r live-record-binary] [-t minutes] [format-configuration]" echo - echo " -a abort/recovery testing (defaults to off)" + echo " -a add configuration for abort/recovery testing (defaults to off)" echo " -b binary format binary (defaults to "./t")" echo " -c config format configuration file (defaults to CONFIG.stress)" - echo " -e envvar Environment variable setting (default to none)" echo " -E skip known errors (defaults to off)" + echo " -e envvar Environment variable setting (default to none)" echo " -F quit on first failure (defaults to off)" echo " -h home run directory (defaults to .)" echo " -j parallel jobs to execute in parallel (defaults to 8)" echo " -n total total jobs to execute (defaults to no limit)" - echo " -R run timing stress split test configurations (defaults to off)" + echo " -R add configuration for randomized split stress (defaults to none)" echo " -r binary record with UndoDB binary (defaults to no recording)" echo " -S run smoke-test configurations (defaults to off)" echo " -t minutes minutes to run (defaults to no limit)" echo " -v verbose output (defaults to off)" - echo " -- separates $name arguments from format arguments" + echo " -- separates $name arguments from additional format arguments" exit 1 } # Smoke-tests. -smoke_base_1="data_source=table rows=100000 threads=6 timer=4" +smoke_base_1="runs.source=table rows=100000 threads=6 timer=4" smoke_base_2="$smoke_base_1 leaf_page_max=9 internal_page_max=9" smoke_list=( # Three access methods. @@ -49,14 +49,14 @@ smoke_list=( # "$smoke_base_1 file_type=fix" # "$smoke_base_1 file_type=var" - # Huffman key/value encoding. + # Huffman value encoding. "$smoke_base_1 file_type=row huffman_value=1" # Temporarily disabled # "$smoke_base_1 file_type=var huffman_value=1" # LSM # Temporarily disabled - # "$smoke_base_1 file_type=row data_source=lsm" + # "$smoke_base_1 file_type=row runs.source=lsm" # Force the statistics server. "$smoke_base_1 file_type=row statistics_server=1" @@ -74,16 +74,16 @@ build="" config="CONFIG.stress" first_failure=0 format_args="" +format_binary="./t" home="." +live_record_binary="" minutes=0 parallel_jobs=8 -live_record_binary="" skip_errors=0 smoke_test=0 -timing_stress_split_test=0 +stress_split_test=0 total_jobs=0 verbose=0 -format_binary="./t" while :; do case "$1" in @@ -96,12 +96,12 @@ while :; do -c) config="$2" shift ; shift ;; - -e) - export "$2" - shift ; shift ;; -E) skip_errors=1 shift ;; + -e) + export "$2" + shift ; shift ;; -F) first_failure=1 shift ;; @@ -123,7 +123,7 @@ while :; do } shift ; shift ;; -R) - timing_stress_split_test=1 + stress_split_test=1 shift ;; -r) live_record_binary="$2" @@ -394,7 +394,7 @@ resolve() # Everything is a table unless explicitly a file. uri="table:wt" - grep 'data_source=file' $dir/CONFIG > /dev/null && uri="file:wt" + grep 'runs.source=file' $dir/CONFIG > /dev/null && uri="file:wt" # Use the wt utility to recover & verify the object. if $($wt_binary -m -R -h $dir verify $uri >> $log 2>&1); then @@ -474,25 +474,21 @@ format() dir="$home/RUNDIR.$count_jobs" log="$dir.log" + args="" if [[ $smoke_test -ne 0 ]]; then args=${smoke_list[$smoke_next]} smoke_next=$(($smoke_next + 1)) - echo "$name: starting smoke-test job in $dir ($(date))" - elif [[ $timing_stress_split_test -ne 0 ]]; then - args=$format_args - for k in {1..7}; do - args+=" timing_stress_split_$k=$(($RANDOM%2))" + fi + if [[ $abort_test -ne 0 ]]; then + args+=" format.abort=1" + fi + if [[ $stress_split_test -ne 0 ]]; then + for k in {1..8}; do + args+=" stress_split_$k=$(($RANDOM%2))" done - echo "$name: starting timing-stress-split job in $dir ($(date))" - else - args=$format_args - - # If abort/recovery testing is configured, do it 5% of the time. - [[ $abort_test -ne 0 ]] && - [[ $(($count_jobs % 20)) -eq 0 ]] && args="$args format.abort=1" - - echo "$name: starting job in $dir ($(date))" fi + args+=" $format_args" + echo "$name: starting job in $dir ($(date))" # If we're using UndoDB, append our default arguments. # diff --git a/src/third_party/wiredtiger/test/format/recover.sh b/src/third_party/wiredtiger/test/format/recover.sh deleted file mode 100644 index 6f65db73eb5..00000000000 --- a/src/third_party/wiredtiger/test/format/recover.sh +++ /dev/null @@ -1,52 +0,0 @@ -#! /bin/sh - -# Ulimit: don't drop core. -ulimit -c 0 - -# Timer: how many minutes format runs before aborting. -timer=2 - -# Runs: set to 0 to run infinitely. -runs=1000 -if test "$#" -gt "0"; then - runs=$1 -fi - -# Config: additional test/format configuration -config= - -# Assumes we're running in build_*/test/format directory. -tcmd=./t -wtcmd=../../wt -rundir2=RUNDIR.SAVE -count=0 -while true; do - count=`expr $count + 1` - if test $runs -eq 0; then - echo "recovery test: $count" - else - if test $count -gt $runs; then - exit 0 - fi - echo "recovery test: $count of $runs" - fi - - rm -rf $rundir2 - $tcmd $config -q format.abort=1 logging=1 runs.timer=$timer - - # Save a copy of the database directory exactly as it was at the crash. - cp -rp RUNDIR $rundir2 - - # - # Everything is a table unless explicitly a file. - # - isfile=`grep data_source RUNDIR/CONFIG | grep -c file || exit 0` - if test "$isfile" -ne 0; then - uri="file:wt" - else - uri="table:wt" - fi - - # We know we aborted, so force recovery to run. - $wtcmd -m -R -h RUNDIR verify $uri || exit 1 -done diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 3ea620e638e..ea8cb43ac1c 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -204,7 +204,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) + self.session.create(uri, 'key_format={},value_format=S,verbose=(write_timestamp),write_timestamp_usage=key_consistent,assert=(write_timestamp=on)'.format(self.key_format)) # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) @@ -438,5 +438,14 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): c.close() ''' + # Confirm that rolling back after preparing doesn't fire an assertion. + c = self.session.open_cursor(uri) + self.session.begin_transaction() + c[key_ts6] = 'value24' + self.session.prepare_transaction( + 'prepare_timestamp=' + self.timestamp_str(24)) + self.session.rollback_transaction() + c.close() + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug022.py b/src/third_party/wiredtiger/test/suite/test_bug022.py index 31fb2387662..8e7b355133e 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug022.py +++ b/src/third_party/wiredtiger/test/suite/test_bug022.py @@ -30,27 +30,37 @@ # Testing that we don't allow modifies on top of tombstone updates. import wiredtiger, wttest +from wtscenario import make_scenarios class test_bug022(wttest.WiredTigerTestCase): uri = 'file:test_bug022' conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def get_key(self, i): + return str(i) if self.usestrings else i + def test_apply_modifies_on_onpage_tombstone(self): - self.session.create(self.uri, 'key_format=S,value_format=S') + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) value = 'a' * 500 for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value + cursor[self.get_key(i)] = value self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply tombstones for every key. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) cursor.remove() self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) @@ -59,11 +69,11 @@ class test_bug022(wttest.WiredTigerTestCase): # Now try to apply a modify on top of the tombstone at timestamp 3. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 0, 100)]), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() # Check that the tombstone is visible. for i in range(1, 10000): - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_dump01.py b/src/third_party/wiredtiger/test/suite/test_dump01.py new file mode 100644 index 00000000000..cd6de8c7f62 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_dump01.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# [TEST_TAGS] +# wt_util +# [END_TAGS] + +import os, shutil +import wiredtiger, wttest + +from suite_subprocess import suite_subprocess + +# test_pretty_hex_dump +# Utilities: wt dump +# Test the dump utility with pretty hex flag +class test_pretty_hex_dump(wttest.WiredTigerTestCase, suite_subprocess): + table_format = 'key_format=i,value_format=u' + uri = 'table:test_dump' + + pretty_dump_file = 'pretty_dump.out' + pretty_hex_dump_file = 'pretty_hex_dump.out' + hex_dump_file = 'hex.out' + + pretty_hex_format = 'Format=print hex\n' + data_header = 'Data\n' + + def get_bytes(self, i, len): + """ + Return a pseudo-random, but predictable string that uses all characters + """ + ret = b'' + for j in range(0, len // 3): + k = i + j + ret += bytes([k%255 + 1, (k*3)%255 + 1, (k*7)%255 + 1]) + return ret + bytes([0]) # Add a final null byte + + def populate_table(self, uri): + """ + Populate test_dump table + """ + cursor = self.session.open_cursor(uri, None, None) + for i in range(2, 15): + val_bytes = self.get_bytes(i*i, i*i) + cursor[i] = (val_bytes) + cursor.close() + + def test_dump(self): + """ + Generates test table with byte array values. After that dumps the table in hex, pretty and + pretty hex formats. And then valudates pretty hex dump. + """ + # Create three dumps: hex, pretty and pretty_hex + self.session.create(self.uri, self.table_format) + self.populate_table(self.uri) + + self.runWt(['dump', '-x', self.uri], outfilename=self.hex_dump_file) + self.runWt(['dump', '-p', self.uri], outfilename=self.pretty_dump_file) + self.runWt(['dump', '-px', self.uri], outfilename=self.pretty_hex_dump_file) + + # Validate pretty hex + hex = open(self.hex_dump_file).readlines() + pretty = open(self.pretty_dump_file).readlines() + pretty_hex = open(self.pretty_hex_dump_file).readlines() + + # First validate number of lines the dumps + self.assertEqual(True, len(pretty) == len(pretty_hex), + 'Pretty and pretty_hex output must have the same numbler of lines.') + self.assertEqual(True, len(hex) == len(pretty_hex), + 'Hex and pretty_hex output must have the same numbler of lines.') + + # Next analyse the pretty hex dump line by line + data_started = False + value_line = False + for h, p, px in zip(hex, pretty, pretty_hex): + if data_started: + # Data section started + if value_line: + # Test values + self.assertEqual(True, h == px, + 'Hex and pretty_hex values must match!\n' + 'Hex: ' + h + 'Pretty_hex: ' + px) + else: + # Test keys + self.assertEqual(True, p == px, + 'Pretty and pretty_hex keys must match!\n' + 'Pretty: ' + p + 'Pretty_hex: ' + px) + + value_line = not value_line + else: + if p.startswith('Format='): + # The two dumps must have appropriate format string + self.assertEqual(True, px == self.pretty_hex_format, + 'Format for pretty_hex dump is: ' + px + ' Expected: ' + self.pretty_hex_format) + else: + # The two dump lines must match + self.assertEqual(True, p == px, + 'Dump lines differ!\n' + 'Pretty: ' + p + ', Pretty_hex: ' + px) + + # Test if data sectio has started + if not data_started and p == self.data_header: + self.assertEqual(True, p == px, + 'Data section starts at different lines.\n' + 'Pretty: ' + p + 'Pretty_hex: ' + px) + self.assertEqual(True, h == px, + 'Data section starts at different lines.\n' + 'Hex: ' + h + 'Pretty_hex: ' + px) + data_started = True + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index deea2f3dfc5..ea3fe771169 100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -28,6 +28,7 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest +from wtscenario import make_scenarios # test_durable_ts03.py # Check that the checkpoint honors the durable timestamp of updates. @@ -35,11 +36,17 @@ class test_durable_ts03(wttest.WiredTigerTestCase): conn_config = 'cache_size=10MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_durable_ts03(self): # Create a table. uri = 'table:test_durable_ts03' nrows = 3000 - self.session.create(uri, 'key_format=i,value_format=u') + self.session.create(uri, 'key_format={},value_format=u'.format(self.key_format)) valueA = b"aaaaa" * 100 valueB = b"bbbbb" * 100 valueC = b"ccccc" * 100 @@ -51,7 +58,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): # Load the data into the table. session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueA session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) @@ -65,7 +72,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): # Update all the values within transaction. Commit the transaction with # a durable timestamp newer than the stable timestamp. cursor = session.open_cursor(uri, None) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueB session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) @@ -105,7 +112,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEqual(value, valueA) self.assertEquals(cursor.reset(), 0) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueC session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(220)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py index b4e65f183b0..de9a7475817 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare06.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -90,24 +90,18 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): s_reader = self.conn.open_session() s_reader.begin_transaction('read_timestamp=' + self.timestamp_str(40)) - # It is illegal to set the prepare timestamp as earlier than an active - # read timestamp even with roundup_timestamps settings. This is only + # It is legal to set the prepare timestamp earlier than an active + # read timestamp with roundup_timestamps settings. This is only # checked in diagnostic builds. if wiredtiger.diagnostic_build(): self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(10)), - "/must be greater than the latest active read timestamp/") + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)), 0) self.session.rollback_transaction() - # It is illegal to set the prepare timestamp the same as an active read - # timestamp even with roundup_timestamps settings. + # It is legal to set the prepare timestamp the same as an active read + # timestamp with roundup_timestamps settings. self.session.begin_transaction('roundup_timestamps=(prepared=true)') - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(40)), - "/must be greater than the latest active read timestamp/") + self.assertEqual(self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)), 0) self.session.rollback_transaction() ''' @@ -144,5 +138,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): s_reader.commit_transaction() + self.ignoreStdoutPatternIfExists("Skip checking prepare timestamp") + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index 5593c75f3ff..b12bf3d12a4 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -30,7 +30,10 @@ import os, wiredtiger, wttest FileSystem = wiredtiger.FileSystem # easy access to constants # test_tiered06.py -# Test the local storage source. +# Test the local storage source's file system implementation. +# Note that the APIs we are testing are not meant to be used directly +# by any WiredTiger application, these APIs are used internally. +# However, it is useful to do tests of this API independently. class test_tiered06(wttest.WiredTigerTestCase): # Load the local store extension. def conn_extensions(self, extlist): @@ -64,16 +67,26 @@ class test_tiered06(wttest.WiredTigerTestCase): # The object doesn't exist yet. self.assertFalse(fs.fs_exist(session, 'foobar')) - fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_create) + # We cannot use the file system to create files, it is readonly. + # So use python I/O to build up the file. + f = open('foobar', 'wb') - # Just like a regular file system, the object exists now. - self.assertTrue(fs.fs_exist(session, 'foobar')) + # The object still doesn't exist yet. + self.assertFalse(fs.fs_exist(session, 'foobar')) outbytes = ('MORE THAN ENOUGH DATA\n'*100000).encode() - fh.fh_write(session, 0, outbytes) + f.write(outbytes) + f.close() - # The object exists after close - fh.close(session) + # Nothing is in the directory list until a flush. + self.assertEquals(fs.fs_directory_list(session, '', ''), []) + + # Flushing moves the file into the file system + local.ss_flush(session, fs, 'foobar', 'foobar', None) + local.ss_flush_finish(session, fs, 'foobar', 'foobar', None) + + # The object exists now. + self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) self.assertTrue(fs.fs_exist(session, 'foobar')) fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) @@ -90,39 +103,14 @@ class test_tiered06(wttest.WiredTigerTestCase): fh.fh_lock(session, False) fh.close(session) - # Nothing is in the directory list until a flush. - self.assertEquals(fs.fs_directory_list(session, '', ''), []) - - fh = fs.fs_open_file(session, 'zzz', FileSystem.open_file_type_data, FileSystem.open_create) - - # Sync merely syncs to the local disk. - fh.fh_sync(session) - fh.close(session) # zero length - self.assertEquals(sorted(fs.fs_directory_list(session, '', '')), []) - - # See that we can rename objects. - fs.fs_rename(session, 'zzz', 'yyy', 0) - self.assertEquals(sorted(fs.fs_directory_list(session, '', '')), []) - - # See that we can remove objects. - fs.fs_remove(session, 'yyy', 0) - - # Nothing is in the directory list until a flush. - self.assertEquals(fs.fs_directory_list(session, '', ''), []) - - # Flushing moves the file. - local.ss_flush(session, fs, 'foobar', 'foobar', None) - local.ss_flush_finish(session, fs, 'foobar', 'foobar', None) - self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) - # Files that have been flushed cannot be manipulated. - with self.expectedStderrPattern('foobar: rename of flushed file not allowed'): + with self.expectedStderrPattern('foobar: rename of file not supported'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: fs.fs_rename(session, 'foobar', 'barfoo', 0)) self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) # Files that have been flushed cannot be manipulated through the custom file system. - with self.expectedStderrPattern('foobar: remove of flushed file not allowed'): + with self.expectedStderrPattern('foobar: remove of file not supported'): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: fs.fs_remove(session, 'foobar', 0)) self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar']) @@ -136,14 +124,17 @@ class test_tiered06(wttest.WiredTigerTestCase): session = self.session local = self.get_local_storage_source() + cachedir = ("objects_cache") os.mkdir("objects") - fs = local.ss_customize_file_system(session, "./objects", "Secret", None) + os.mkdir("objects_cache") + fs = local.ss_customize_file_system( + session, "./objects", "Secret", "cache_directory=" + cachedir) # We call these 4K chunks of data "blocks" for this test, but that doesn't # necessarily relate to WT block sizing. nblocks = 1000 block_size = 4096 - fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_create) + f = open('abc', 'wb') # blocks filled with 'a', etc. a_block = ('a' * block_size).encode() @@ -153,47 +144,63 @@ class test_tiered06(wttest.WiredTigerTestCase): # write all blocks as 'a', but in reverse order for pos in range(file_size - block_size, 0, -block_size): - fh.fh_write(session, pos, a_block) + f.seek(pos) + f.write(a_block) # write the even blocks as 'b', forwards for pos in range(0, file_size, block_size * 2): - fh.fh_write(session, pos, b_block) + f.seek(pos) + f.write(b_block) # write every third block as 'c', backwards for pos in range(file_size - block_size, 0, -block_size * 3): - fh.fh_write(session, pos, c_block) - fh.close(session) + f.seek(pos) + f.write(c_block) + f.close() + + # Flushing moves the file into the file system + local.ss_flush(session, fs, 'abc', 'abc', None) + local.ss_flush_finish(session, fs, 'abc', 'abc', None) + + # Use the file system to open and read the file. + # We do this twice, and between iterations, we remove the cached file to make sure + # it is copied back from the bucket directory. + # + # XXX: this uses knowledge of the implementation, but at the current time, + # we don't have a way via the API to "age out" a file from the cache. + for i in range(0, 2): + in_block = bytes(block_size) + fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_readonly) + + # Do some spot checks, reading non-sequentially + fh.fh_read(session, 500 * block_size, in_block) # divisible by 2, not 3 + self.assertEquals(in_block, b_block) + fh.fh_read(session, 333 * block_size, in_block) # divisible by 3, not 2 + self.assertEquals(in_block, c_block) + fh.fh_read(session, 401 * block_size, in_block) # not divisible by 2 or 3 + self.assertEquals(in_block, a_block) + + # Read the whole file, backwards checking to make sure + # each block was written correctly. + for block_num in range(nblocks - 1, 0, -1): + pos = block_num * block_size + fh.fh_read(session, pos, in_block) + if block_num % 3 == 0: + self.assertEquals(in_block, c_block) + elif block_num % 2 == 0: + self.assertEquals(in_block, b_block) + else: + self.assertEquals(in_block, a_block) + fh.close(session) + os.remove(os.path.join(cachedir, 'abc')) - in_block = bytes(block_size) - fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_readonly) - - # Do some spot checks, reading non-sequentially - fh.fh_read(session, 500 * block_size, in_block) # divisible by 2, not 3 - self.assertEquals(in_block, b_block) - fh.fh_read(session, 333 * block_size, in_block) # divisible by 3, not 2 - self.assertEquals(in_block, c_block) - fh.fh_read(session, 401 * block_size, in_block) # not divisible by 2 or 3 - self.assertEquals(in_block, a_block) - - # Read the whole file, backwards checking to make sure - # each block was written correctly. - for block_num in range(nblocks - 1, 0, -1): - pos = block_num * block_size - fh.fh_read(session, pos, in_block) - if block_num % 3 == 0: - self.assertEquals(in_block, c_block) - elif block_num % 2 == 0: - self.assertEquals(in_block, b_block) - else: - self.assertEquals(in_block, a_block) - fh.close(session) local.terminate(session) def create_with_fs(self, fs, fname): session = self.session - fh = fs.fs_open_file(session, fname, FileSystem.open_file_type_data, FileSystem.open_create) - fh.fh_write(session, 0, 'some stuff'.encode()) - fh.close(session) + f = open(fname, 'wb') + f.write('some stuff'.encode()) + f.close() objectdir1 = "./objects1" objectdir2 = "./objects2" @@ -298,7 +305,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # A flush copies to the cloud, nothing is removed. local.ss_flush(session, fs1, 'beagle.wt', 'beagle.wtobj') self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) - self.check_dirlist(fs1, '', []) + self.check_dirlist(fs1, '', ['beagle']) self.check_dirlist(fs2, '', []) self.check_caches([], []) self.check_objects(['beagle'], []) @@ -311,7 +318,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # It's okay to flush again, nothing changes local.ss_flush(session, fs1, 'beagle.wt', 'beagle.wtobj') self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) - self.check_dirlist(fs1, '', []) + self.check_dirlist(fs1, '', ['beagle']) self.check_dirlist(fs2, '', []) self.check_caches([], []) self.check_objects(['beagle'], []) @@ -333,13 +340,13 @@ class test_tiered06(wttest.WiredTigerTestCase): local.ss_flush_finish(session, fs1, 'bat.wt', 'bat.wtobj') self.check_home(['bird', 'bison', 'cougar', 'coyote', 'cub']) - self.check_dirlist(fs1, '', ['beagle', 'bat']) - self.check_dirlist(fs2, '', ['cat']) + self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison']) + self.check_dirlist(fs2, '', ['cat', 'cub']) self.check_caches(['beagle', 'bat'], ['cat']) self.check_objects(['beagle', 'bat', 'bison'], ['cat', 'cub']) # Test directory listing prefixes - self.check_dirlist(fs1, '', ['beagle', 'bat']) + self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison']) self.check_dirlist(fs1, 'ba', ['bat']) self.check_dirlist(fs1, 'be', ['beagle']) self.check_dirlist(fs1, 'x', []) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index 4af2004c162..be6dafc5c38 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -45,6 +45,10 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): nentries = 10 table_cnt = 3 + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] types = [ ('all', dict(use_stable='false', run_wt=0)), ('all+wt', dict(use_stable='false', run_wt=1)), @@ -56,7 +60,7 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): ('stable+wt', dict(use_stable='true', run_wt=1)), ('stable+wt2', dict(use_stable='true', run_wt=2)), ] - scenarios = make_scenarios(types) + scenarios = make_scenarios(key_format_values, types) def data_and_checkpoint(self): # @@ -64,10 +68,11 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): # Add data to each of them separately and checkpoint so that each one # has a different stable timestamp. # - self.session.create(self.oplog_uri, 'key_format=i,value_format=i') - self.session.create(self.coll1_uri, 'key_format=i,value_format=i,log=(enabled=false)') - self.session.create(self.coll2_uri, 'key_format=i,value_format=i,log=(enabled=false)') - self.session.create(self.coll3_uri, 'key_format=i,value_format=i,log=(enabled=false)') + basecfg = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.oplog_uri, basecfg) + self.session.create(self.coll1_uri, basecfg + ',log=(enabled=false)') + self.session.create(self.coll2_uri, basecfg + ',log=(enabled=false)') + self.session.create(self.coll3_uri, basecfg + ',log=(enabled=false)') c_op = self.session.open_cursor(self.oplog_uri) c = [] c.append(self.session.open_cursor(self.coll1_uri)) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp11.py b/src/third_party/wiredtiger/test/suite/test_timestamp11.py index c4b4210df43..d5998bdc212 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp11.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp11.py @@ -32,14 +32,24 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + def test_timestamp_range(self): base = 'timestamp11' uri = 'file:' + base - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + + key = 'key' if self.usestrings else 1 + key2 = 'key2' if self.usestrings else 2 # Test that mixed timestamp usage where some transactions use timestamps # and others don't behave in the expected way. @@ -49,8 +59,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(2)) - c['key'] = 'value2' - c['key2'] = 'value2' + c[key] = 'value2' + c[key2] = 'value2' self.session.commit_transaction() c.close() @@ -62,13 +72,13 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(5)) - c['key'] = 'value5' + c[key] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key2'] = 'valueNOTS' + c[key2] = 'valueNOTS' self.session.commit_transaction() c.close() @@ -85,15 +95,15 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - self.assertEquals(c['key'], 'value2') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'value2') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + stable_ts) - self.assertEquals(c['key'], 'value2') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'value2') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() @@ -104,13 +114,13 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(5)) - c['key2'] = 'value5' + c[key2] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key'] = 'valueNOTS' + c[key] = 'valueNOTS' self.session.commit_transaction() c.close() @@ -119,8 +129,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # Without a timestamp. We should see the latest value for each. c = self.session.open_cursor(uri) self.session.begin_transaction() - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'value5') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'value5') self.session.commit_transaction() c.close() @@ -128,8 +138,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # value at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + stable_ts) - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() @@ -138,8 +148,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # we inserted at timestamp 5 after the non-timestamped insert. c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'value5') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'value5') self.session.commit_transaction() c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp12.py b/src/third_party/wiredtiger/test/suite/test_timestamp12.py index e54ed923474..5ccab50ce05 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp12.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp12.py @@ -38,12 +38,16 @@ class test_timestamp12(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' coll_uri = 'table:collection12' oplog_uri = 'table:oplog12' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] closecfg = [ ('dfl', dict(close_cfg='', all_expected=False)), ('use_stable', dict(close_cfg='use_timestamp=true', all_expected=False)), ('all_dirty', dict(close_cfg='use_timestamp=false', all_expected=True)), - ] - scenarios = make_scenarios(closecfg) + ] + scenarios = make_scenarios(key_format_values, closecfg) def verify_expected(self, op_exp, coll_exp): c_op = self.session.open_cursor(self.oplog_uri) @@ -67,8 +71,9 @@ class test_timestamp12(wttest.WiredTigerTestCase): # Add data to each of them separately and checkpoint so that each one # has a different stable timestamp. # - self.session.create(self.oplog_uri, 'key_format=i,value_format=i') - self.session.create(self.coll_uri, 'key_format=i,value_format=i,log=(enabled=false)') + basecfg = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.oplog_uri, basecfg) + self.session.create(self.coll_uri, basecfg + ',log=(enabled=false)') c_op = self.session.open_cursor(self.oplog_uri) c_coll = self.session.open_cursor(self.coll_uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py index 8d6e753bf4d..3b4c4bf40ae 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp14.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -40,6 +40,12 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_all_durable_old(self): # This test was originally for testing the all_committed timestamp. # In the absence of prepared transactions, all_durable is identical to @@ -47,8 +53,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): all_durable_uri = self.uri + '_all_durable' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(all_durable_uri, 'key_format=i,value_format=i') - session2.create(all_durable_uri, 'key_format=i,value_format=i') + session1.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) # Scenario 0: No commit timestamp has ever been specified therefore # There is no all_durable timestamp and we will get an error @@ -132,8 +138,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): oldest_reader_uri = self.uri + '_oldest_reader_pinned' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(oldest_reader_uri, 'key_format=i,value_format=i') - session2.create(oldest_reader_uri, 'key_format=i,value_format=i') + session1.create(oldest_reader_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(oldest_reader_uri, 'key_format={},value_format=i'.format(self.key_format)) # Nothing is reading so there is no oldest reader. self.assertRaisesException(wiredtiger.WiredTigerError, @@ -190,7 +196,7 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): def test_pinned_oldest(self): pinned_oldest_uri = self.uri + 'pinned_oldest' session1 = self.setUpSessionOpen(self.conn) - session1.create(pinned_oldest_uri, 'key_format=i,value_format=i') + session1.create(pinned_oldest_uri, 'key_format={},value_format=i'.format(self.key_format)) # Confirm no oldest timestamp exists. self.assertRaisesException(wiredtiger.WiredTigerError, lambda: self.conn.query_timestamp('get=oldest')) @@ -241,7 +247,7 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): def test_all_durable(self): all_durable_uri = self.uri + '_all_durable' session1 = self.setUpSessionOpen(self.conn) - session1.create(all_durable_uri, 'key_format=i,value_format=i') + session1.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) # Since this is a non-prepared transaction, we'll be using the commit # timestamp when calculating all_durable since it's implied that they're @@ -329,8 +335,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): all_uri = self.uri + 'pinned_oldest' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(all_uri, 'key_format=i,value_format=i') - session2.create(all_uri, 'key_format=i,value_format=i') + session1.create(all_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(all_uri, 'key_format={},value_format=i'.format(self.key_format)) cur1 = session1.open_cursor(all_uri) cur2 = session2.open_cursor(all_uri) # Set up oldest timestamp. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp16.py b/src/third_party/wiredtiger/test/suite/test_timestamp16.py index f74557ad2c6..2485a14be12 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp16.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp16.py @@ -34,7 +34,6 @@ import random from suite_subprocess import suite_subprocess import wiredtiger, wttest -from wtscenario import make_scenarios class test_timestamp16(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp16' diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp17.py b/src/third_party/wiredtiger/test/suite/test_timestamp17.py index 3c2453b0aa2..fe4ec606d23 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp17.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp17.py @@ -43,8 +43,14 @@ class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_inconsistent_timestamping(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) self.session.begin_transaction() cur1 = self.session.open_cursor(self.uri) cur1[1] = 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp18.py b/src/third_party/wiredtiger/test/suite/test_timestamp18.py index c7f20dbd9a4..b41fc5ccb34 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp18.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp18.py @@ -41,15 +41,23 @@ from wtscenario import make_scenarios class test_timestamp18(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] non_ts_writes = [ ('insert', dict(delete=False)), ('delete', dict(delete=True)), ] - scenarios = make_scenarios(non_ts_writes) + scenarios = make_scenarios(key_format_values, non_ts_writes) + + def get_key(self, i): + return str(i) if self.usestrings else i def test_ts_writes_with_non_ts_write(self): uri = 'table:test_timestamp18' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -61,17 +69,17 @@ class test_timestamp18(wttest.WiredTigerTestCase): # A series of timestamped writes on each key. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Add a non-timestamped delete. @@ -80,10 +88,10 @@ class test_timestamp18(wttest.WiredTigerTestCase): for i in range(1, 10000): if i % 2 == 0: if self.delete: - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) cursor.remove() else: - cursor[str(i)] = value4 + cursor[self.get_key(i)] = value4 self.session.checkpoint() @@ -94,16 +102,16 @@ class test_timestamp18(wttest.WiredTigerTestCase): # invisible. if i % 2 == 0: if self.delete: - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: - self.assertEqual(cursor[str(i)], value4) + self.assertEqual(cursor[self.get_key(i)], value4) # Otherwise, expect one of the timestamped writes. else: if ts == 2: - self.assertEqual(cursor[str(i)], value1) + self.assertEqual(cursor[self.get_key(i)], value1) elif ts == 3: - self.assertEqual(cursor[str(i)], value2) + self.assertEqual(cursor[self.get_key(i)], value2) else: - self.assertEqual(cursor[str(i)], value3) + self.assertEqual(cursor[self.get_key(i)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py index 9f5b71df843..e271c8f1145 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp19.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -30,15 +30,22 @@ # Use the oldest timestamp in the metadata as the oldest timestamp on restart. import wiredtiger, wttest from wtdataset import SimpleDataSet +from wtscenario import make_scenarios class test_timestamp19(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def updates(self, uri, value, ds, nrows, commit_ts): session = self.session cursor = session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) @@ -46,11 +53,11 @@ class test_timestamp19(wttest.WiredTigerTestCase): def test_timestamp(self): uri = "table:test_timestamp19" - create_params = 'value_format=S,key_format=i' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_params) ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() nrows = 1000 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp20.py b/src/third_party/wiredtiger/test/suite/test_timestamp20.py index c4edcbfe592..7bfeb670004 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp20.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp20.py @@ -27,6 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_timestamp20.py # Exercise fixing up of out-of-order updates in the history store. @@ -34,9 +35,18 @@ class test_timestamp20(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def get_key(self, i): + return str(i) if self.usestrings else i + def test_timestamp20_standard(self): uri = 'table:test_timestamp20' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -48,17 +58,17 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) old_reader_session = self.conn.open_session() @@ -69,19 +79,19 @@ class test_timestamp20(wttest.WiredTigerTestCase): # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value4 + cursor[self.get_key(i)] = value4 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() - cursor[str(i)] = value5 + cursor[self.get_key(i)] = value5 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): - self.assertEqual(cursor[str(i)], value4) + self.assertEqual(cursor[self.get_key(i)], value4) self.session.rollback_transaction() for i in range(1, 10000): - self.assertEqual(old_reader_cursor[str(i)], value2) + self.assertEqual(old_reader_cursor[self.get_key(i)], value2) old_reader_session.rollback_transaction() # In this test we're using modifies since they are more sensitive to corruptions. @@ -90,7 +100,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # the conversion to a Python string. def test_timestamp20_modify(self): uri = 'table:test_timestamp20' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -101,19 +111,19 @@ class test_timestamp20(wttest.WiredTigerTestCase): # Apply the base value. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Now apply a series of modifies. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) @@ -129,7 +139,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # This will be the end of the chain of modifies. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) @@ -137,17 +147,17 @@ class test_timestamp20(wttest.WiredTigerTestCase): # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) # Open up a new transaction and read at 30. # We shouldn't be able to see past 5 due to txnid visibility. self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): - self.assertEqual(cursor[str(i)], value2) + self.assertEqual(cursor[self.get_key(i)], value2) self.session.rollback_transaction() # Put together expected value. @@ -157,5 +167,5 @@ class test_timestamp20(wttest.WiredTigerTestCase): # On the other hand, this older transaction SHOULD be able to read past the 5. for i in range(1, 10000): - self.assertEqual(old_reader_cursor[str(i)], expected) + self.assertEqual(old_reader_cursor[self.get_key(i)], expected) old_reader_session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index e02dbf99e09..b26b844dd32 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -31,6 +31,7 @@ import wiredtiger, wttest, re, suite_random from wtdataset import SimpleDataSet from contextlib import contextmanager +from wtscenario import make_scenarios class test_timestamp22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' @@ -46,6 +47,12 @@ class test_timestamp22(wttest.WiredTigerTestCase): SUCCESS = 'success' FAILURE = 'failure' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + # Control execution of an operation, looking for exceptions and error messages. # Usage: # with self.expect(self.FAILURE, 'some operation'): @@ -240,7 +247,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): msg = 'inserts with commit config(' + commit_config + ')' try: - for i in range(0, self.nrows): + for i in range(1, self.nrows + 1): needs_rollback = False if self.do_illegal(): # Illegal outside of transaction @@ -380,14 +387,14 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: iterations = 1000 - create_params = 'value_format=S,key_format=i' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(self.uri, create_params) self.set_global_timestamps(1, 1, -1, -1) # Create tables with no entries ds = SimpleDataSet( - self, self.uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, self.uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') # We do a bunch of iterations, doing transactions, prepare, and global timestamp calls # with timestamps that are sometimes valid, sometimes not. We use the iteration number @@ -430,7 +437,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Make sure the resulting rows are what we expect. cursor = self.session.open_cursor(self.uri) - expect_key = 0 + expect_key = 1 expect_value = self.commit_value for k,v in cursor: self.assertEquals(k, expect_key) @@ -440,7 +447,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Although it's theoretically possible to never successfully update a single row, # with a large number of iterations that should never happen. I'd rather catch # a test code error where we mistakenly don't update any rows. - self.assertGreater(expect_key, 0) + self.assertGreater(expect_key, 1) cursor.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp23.py b/src/third_party/wiredtiger/test/suite/test_timestamp23.py new file mode 100644 index 00000000000..60d97cf78fb --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp23.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_timestamp23.py +# +# delete keys repeatedly at successive timestamps +class test_timestamp23(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + + def test_timestamp(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:timestamp23' + ds = SimpleDataSet( + self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + key = 5 + value_1 = 'a' * 500 + value_2 = 'b' * 500 + value_3 = 'c' * 500 + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + cursor = self.session.open_cursor(ds.uri) + + # Write two values at timestamp 10. We'll muck with the first value + # and use the second to reference the page for eviction. + self.session.begin_transaction('read_timestamp=10') + cursor[key] = value_1 + cursor[key+1] = value_2 + self.session.commit_transaction('commit_timestamp=11') + + # Delete the first value at timestamp 20. + self.session.begin_transaction('read_timestamp=20') + cursor.set_key(key) + cursor.remove() + self.session.commit_transaction('commit_timestamp=21') + + # Put it back at timestamp 30. + self.session.begin_transaction('read_timestamp=30') + cursor[key] = value_3 + self.session.commit_transaction('commit_timestamp=31') + + # Delete it again at timestamp 40. + self.session.begin_transaction('read_timestamp=40') + cursor.set_key(key) + cursor.remove() + self.session.commit_transaction('commit_timestamp=41') + + # Evict the page using the second key. + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + v = evict_cursor[key+1] + self.assertEqual(v, value_2) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Create a separate session and a cursor to read the original value at timestamp 12. + session2 = self.conn.open_session() + cursor2 = session2.open_cursor(ds.uri) + session2.begin_transaction('read_timestamp=12') + v = cursor2[key] + self.assertEqual(v, value_1) + + self.session.breakpoint() + + # Now delete the original value. This _should_ cause WT_ROLLBACK, but with a column + # store bug seen and fixed in August 2021, it succeeds, and the resulting invalid + # tombstone will cause reconciliation to assert. (To see this behavior, comment out the + # self.fail call and let the transaction commit.) + try: + cursor2.remove() + self.fail("Conflicting remove did not fail") + session2.commit_transaction('commit_timestamp=50') + except wiredtiger.WiredTigerError as e: + self.assertTrue(wiredtiger.wiredtiger_strerror(wiredtiger.WT_ROLLBACK) in str(e)) + + cursor.close() + cursor2.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/test_coverage.md b/src/third_party/wiredtiger/test/test_coverage.md index bc71f581c09..b186c7ac318 100644 --- a/src/third_party/wiredtiger/test/test_coverage.md +++ b/src/third_party/wiredtiger/test/test_coverage.md @@ -53,4 +53,4 @@ |Truncate||[test_truncate01.py](../test/suite/test_truncate01.py) |Truncate|Prepare|[test_prepare13.py](../test/suite/test_prepare13.py) |Verify|Prepare|[test_prepare_hs03.py](../test/suite/test_prepare_hs03.py), [test_timestamp18.py](../test/suite/test_timestamp18.py) -|Wt Util||[test_backup01.py](../test/suite/test_backup01.py), [test_dump.py](../test/suite/test_dump.py), [test_util11.py](../test/suite/test_util11.py) +|Wt Util||[test_backup01.py](../test/suite/test_backup01.py), [test_dump.py](../test/suite/test_dump.py), [test_dump01.py](../test/suite/test_dump01.py), [test_util11.py](../test/suite/test_util11.py) -- cgit v1.2.1