Import wiredtiger: fe6c936cee7b6d3329e1d4df8da5aa56acfa7abb from branch mongodb-master

ref: 1b63816bab..fe6c936cee
for: 6.0.0

WT-8958       Improvements to ARM and other stress tests on evergreen

17 files changed, 171 insertions, 5 deletions

diff --git a/src/third_party/wiredtiger/cmake/configs/aarch64/darwin/config.cmake b/src/third_party/wiredtiger/cmake/configs/aarch64/darwin/config.cmake
index 6e3ffebaa6a..89f6e9d22de 100644
--- a/src/third_party/wiredtiger/cmake/configs/aarch64/darwin/config.cmake
+++ b/src/third_party/wiredtiger/cmake/configs/aarch64/darwin/config.cmake
@@ -17,3 +17,10 @@ if(has_moutline_atomics)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -moutline-atomics" CACHE STRING "" FORCE)
 endif()
 unset(has_moutline_atomics CACHE)
+
+# Enable ARM Neon SIMD instrinsics when available.
+CHECK_INCLUDE_FILE("arm_neon.h" has_arm_neon)
+if(has_arm_neon)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_ARM_NEON_INTRIN_H" CACHE STRING "" FORCE)
+endif()
+unset(has_arm_neon CACHE)
diff --git a/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake
index a72bfe0f465..cdbe569e359 100644
--- a/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake
+++ b/src/third_party/wiredtiger/cmake/configs/aarch64/linux/config.cmake
@@ -25,3 +25,10 @@ if(has_moutline_atomics)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -moutline-atomics" CACHE STRING "" FORCE)
 endif()
 unset(has_moutline_atomics CACHE)
+
+# Enable ARM Neon SIMD instrinsics when available.
+CHECK_INCLUDE_FILE("arm_neon.h" have_arm_neon)
+if(have_arm_neon)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_ARM_NEON_INTRIN_H" CACHE STRING "" FORCE)
+endif()
+unset(has_arm_neon CACHE)
diff --git a/src/third_party/wiredtiger/cmake/configs/base.cmake b/src/third_party/wiredtiger/cmake/configs/base.cmake
index 9980f5fa5d1..6a02b121308 100644
--- a/src/third_party/wiredtiger/cmake/configs/base.cmake
+++ b/src/third_party/wiredtiger/cmake/configs/base.cmake
@@ -43,6 +43,12 @@ config_bool(
 )
 
 config_bool(
+    NON_BARRIER_DIAGNOSTIC_YIELDS
+    "Don't set a full barrier when yielding threads in diagnostic mode. Requires diagnostic mode to be enabled."
+    DEFAULT OFF
+)
+
+config_bool(
     HAVE_UNITTEST
     "Enable WiredTiger unit tests"
     DEFAULT OFF
@@ -245,6 +251,10 @@ if(HAVE_DIAGNOSTIC AND (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug"))
     endif()
 endif()
 
+if (NON_BARRIER_DIAGNOSTIC_YIELDS AND NOT HAVE_DIAGNOSTIC)
+    message(FATAL_ERROR "`NON_BARRIER_DIAGNOSTIC_YIELDS` can only be enabled when `HAVE_DIAGNOSTIC` is enabled.")
+endif()
+
 if(WT_WIN)
     # Check if we a using the dynamic or static run-time library.
     if(DYNAMIC_CRT)
diff --git a/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in b/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in
index c0b822ff160..4072a12e8f9 100644
--- a/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in
+++ b/src/third_party/wiredtiger/cmake/configs/wiredtiger_config.h.in
@@ -33,6 +33,9 @@
 /* Define to 1 for diagnostic tests. */
 #cmakedefine HAVE_DIAGNOSTIC 1
 
+/* Define to 1 to remove full memory barriers on diagnostic yields. */
+#cmakedefine NON_BARRIER_DIAGNOSTIC_YIELDS 0
+
 /* Define to 1 for unit tests. */
 #cmakedefine HAVE_UNITTEST 0
 
diff --git a/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake
index af930bc808b..f8655792258 100644
--- a/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake
+++ b/src/third_party/wiredtiger/cmake/configs/x86/darwin/config.cmake
@@ -1,3 +1,10 @@
 set(WT_ARCH "x86" CACHE STRING "")
 set(WT_OS "darwin" CACHE STRING "")
 set(WT_POSIX ON CACHE BOOL "")
+
+# Enable x86 SIMD instrinsics when available.
+CHECK_INCLUDE_FILE("x86intrin.h" has_x86intrin)
+if(has_x86intrin)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_X86INTRIN_H" CACHE STRING "" FORCE)
+endif()
+unset(has_x86intrin CACHE)
diff --git a/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake b/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake
index 159c2a9ed70..7bb09eb29ab 100644
--- a/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake
+++ b/src/third_party/wiredtiger/cmake/configs/x86/linux/config.cmake
@@ -9,3 +9,10 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE")
 
 # Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work.
 set(WT_BUFFER_ALIGNMENT_DEFAULT "4096" CACHE STRING "")
+
+# Enable x86 SIMD instrinsics when available.
+CHECK_INCLUDE_FILE("x86intrin.h" has_x86intrin)
+if(has_x86intrin)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_X86INTRIN_H" CACHE STRING "" FORCE)
+endif()
+unset(has_x86intrin CACHE)
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index f6187de00a8..8e94b159ae8 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -515,6 +515,7 @@ WiredTigerPreplog
 WiredTigerTmplog
 WiredTigerTxn
 WithSeeds
+Wmissing
 Wpedantic
 WriteFile
 Wsign
@@ -1199,6 +1200,7 @@ nontransactionally
 noop
 nop
 noraw
+noreturn
 notfound
 notsup
 notused
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index a96d3739efa..b2e6c946fb3 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "1b63816bab2c1d89a7bb96ea18c859b59a74c9f0"
+    "commit": "fe6c936cee7b6d3329e1d4df8da5aa56acfa7abb"
 }
diff --git a/src/third_party/wiredtiger/src/docs/build-posix.dox b/src/third_party/wiredtiger/src/docs/build-posix.dox
index 6a96c19b32b..75f9969e7b9 100644
--- a/src/third_party/wiredtiger/src/docs/build-posix.dox
+++ b/src/third_party/wiredtiger/src/docs/build-posix.dox
@@ -105,6 +105,11 @@ Configure WiredTiger to sleep and wait for a debugger to attach on failure.
 Configure WiredTiger to perform various run-time diagnostic tests.
 <b>DO NOT</b> configure this option in production environments.
 
+@par \c -DNON_BARRIER_DIAGNOSTIC_YIELDS=1
+Configure WiredTiger to not use memory barriers when yielding threads for diagnostic purposes.
+Requires that HAVE_DIAGNOSTIC is also enabled.
+<b>DO NOT</b> configure this option in production environments.
+
 @par \c -DENABLE_LZ4=1
 Configure WiredTiger for <a href="https://github.com/Cyan4973/lz4">LZ4</a>
 compression; see @ref compression for more information.
diff --git a/src/third_party/wiredtiger/src/docs/build-windows.dox b/src/third_party/wiredtiger/src/docs/build-windows.dox
index 759ccfa32d3..918eed85ff0 100644
--- a/src/third_party/wiredtiger/src/docs/build-windows.dox
+++ b/src/third_party/wiredtiger/src/docs/build-windows.dox
@@ -97,6 +97,11 @@ Configure WiredTiger to sleep and wait for a debugger to attach on failure.
 Configure WiredTiger to perform various run-time diagnostic tests.
 <b>DO NOT</b> configure this option in production environments.
 
+@par \c -DNON_BARRIER_DIAGNOSTIC_YIELDS=1
+Configure WiredTiger to not use memory barriers when yielding threads for diagnostic purposes.
+Requires that HAVE_DIAGNOSTIC is also enabled.
+<b>DO NOT</b> configure this option in production environments.
+
 @par \c -DENABLE_LZ4=1
 Configure WiredTiger for <a href="https://github.com/Cyan4973/lz4">LZ4</a>
 compression; see @ref compression for more information.
diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok
index 5ee6ec9d409..144eed9cea2 100644
--- a/src/third_party/wiredtiger/src/docs/spell.ok
+++ b/src/third_party/wiredtiger/src/docs/spell.ok
@@ -37,6 +37,7 @@ DCMAKE
 DENABLE
 DHANDLE
 DHAVE
+DNON
 DONTNEED
 DSPINLOCK
 DWT
diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h
index 9acb8a9d6f0..a59b0c51c6e 100644
--- a/src/third_party/wiredtiger/src/include/error.h
+++ b/src/third_party/wiredtiger/src/include/error.h
@@ -11,7 +11,12 @@
 #define WT_DEBUG_BYTE (0xab)
 
 /* In DIAGNOSTIC mode, yield in places where we want to encourage races. */
-#ifdef HAVE_DIAGNOSTIC
+#if defined HAVE_DIAGNOSTIC && defined NON_BARRIER_DIAGNOSTIC_YIELDS
+#define WT_DIAGNOSTIC_YIELD      \
+    do {                         \
+        __wt_yield_no_barrier(); \
+    } while (0)
+#elif defined HAVE_DIAGNOSTIC && !defined NON_BARRIER_DIAGNOSTIC_YIELDS
 #define WT_DIAGNOSTIC_YIELD \
     do {                    \
         __wt_yield();       \
diff --git a/src/third_party/wiredtiger/src/include/extern_posix.h b/src/third_party/wiredtiger/src/include/extern_posix.h
index cee85e85c42..02fe6a9f1dd 100644
--- a/src/third_party/wiredtiger/src/include/extern_posix.h
+++ b/src/third_party/wiredtiger/src/include/extern_posix.h
@@ -65,6 +65,7 @@ extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visi
 extern void __wt_thread_id(uintmax_t *id) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
 extern void __wt_unmap_file(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session);
 extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
+extern void __wt_yield_no_barrier(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
 
 #ifdef HAVE_UNITTEST
 
diff --git a/src/third_party/wiredtiger/src/include/extern_win.h b/src/third_party/wiredtiger/src/include/extern_win.h
index 2a769ac667d..ab519fd8cc0 100644
--- a/src/third_party/wiredtiger/src/include/extern_win.h
+++ b/src/third_party/wiredtiger/src/include/extern_win.h
@@ -61,6 +61,7 @@ extern void __wt_stream_set_line_buffer(FILE *fp);
 extern void __wt_stream_set_no_buffer(FILE *fp);
 extern void __wt_thread_id(uintmax_t *id);
 extern void __wt_yield(void);
+extern void __wt_yield_no_barrier(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
 
 #ifdef HAVE_UNITTEST
 
diff --git a/src/third_party/wiredtiger/src/os_posix/os_yield.c b/src/third_party/wiredtiger/src/os_posix/os_yield.c
index 9d678ae42c5..3bf6839ba0c 100644
--- a/src/third_party/wiredtiger/src/os_posix/os_yield.c
+++ b/src/third_party/wiredtiger/src/os_posix/os_yield.c
@@ -24,3 +24,29 @@ __wt_yield(void) WT_GCC_FUNC_ATTRIBUTE((visibility("default")))
 
     sched_yield();
 }
+
+/*
+ * If HAVE_DIAGNOSTIC is 0 clang will raise an error when the noreturn attribute is missing, and if
+ * HAVE_DIAGNOSTIC is 1 clang will raise an error when the noreturn attribute is present. This
+ * function should never be called when HAVE_DIAGNOSTIC is 0 so we can ignore the missing-noreturn
+ * warning here.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmissing-noreturn"
+/*
+ * __wt_yield_no_barrier --
+ *     Yield the thread of control. Don't set any memory barriers as this may hide memory
+ *     synchronization errors in the surrounding code. It's not explicitly documented that yielding
+ *     without a memory barrier is safe, so this function should only be used for testing in
+ *     diagnostic mode.
+ */
+void
+__wt_yield_no_barrier(void) WT_GCC_FUNC_ATTRIBUTE((visibility("default")))
+{
+#ifndef HAVE_DIAGNOSTIC
+    __wt_abort(NULL);
+#else
+    sched_yield();
+#endif
+}
+#pragma GCC diagnostic pop
diff --git a/src/third_party/wiredtiger/src/os_win/os_yield.c b/src/third_party/wiredtiger/src/os_win/os_yield.c
index 70615f2b4ae..3e0dd37f808 100644
--- a/src/third_party/wiredtiger/src/os_win/os_yield.c
+++ b/src/third_party/wiredtiger/src/os_win/os_yield.c
@@ -24,3 +24,20 @@ __wt_yield(void)
 
     SwitchToThread();
 }
+
+/*
+ * __wt_yield_no_barrier --
+ *     Yield the thread of control. Don't set any memory barriers as this may hide memory
+ *     synchronization errors in the surrounding code. It's not explicitly documented that yielding
+ *     without a memory barrier is safe, so this function should only be used for testing in
+ *     diagnostic mode.
+ */
+void
+__wt_yield_no_barrier(void) WT_GCC_FUNC_ATTRIBUTE((visibility("default")))
+{
+#ifndef HAVE_DIAGNOSTIC
+    __wt_abort(NULL);
+#else
+    SwitchToThread();
+#endif
+}
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index 6459920f4ac..dbdab1afb3e 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -168,7 +168,7 @@ functions:
             mkdir -p cmake_build
             cd cmake_build
             $CMAKE -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v4_clang.cmake -DCMAKE_C_FLAGS="-ggdb" -DWITH_PIC=1 \
-              -DHAVE_DIAGNOSTIC=1 -DCMAKE_BUILD_TYPE=ASan \
+              -DHAVE_DIAGNOSTIC=1 ${NON_BARRIER_DIAGNOSTIC_YIELDS|} -DCMAKE_BUILD_TYPE=ASan \
               -DHAVE_BUILTIN_EXTENSION_LZ4=1 -DHAVE_BUILTIN_EXTENSION_SNAPPY=1 -DHAVE_BUILTIN_EXTENSION_ZLIB=1 ${configure_python_setting|} \
               -G "${cmake_generator|Ninja}" ../.
           fi
@@ -187,7 +187,7 @@ functions:
             mkdir -p cmake_build
             cd cmake_build
             $CMAKE -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DWITH_PIC=1 \
-              -DHAVE_DIAGNOSTIC=1 -DENABLE_STRICT=1  \
+              -DHAVE_DIAGNOSTIC=1 ${NON_BARRIER_DIAGNOSTIC_YIELDS|} -DENABLE_STRICT=1  \
               -DHAVE_BUILTIN_EXTENSION_LZ4=1 -DHAVE_BUILTIN_EXTENSION_SNAPPY=1 -DHAVE_BUILTIN_EXTENSION_ZLIB=1 ${configure_python_setting|} \
               -G "${cmake_generator|Ninja}" ../.
           fi
@@ -771,6 +771,9 @@ tasks:
       - func: "get project"
       - func: "compile wiredtiger"
         vars:
+          posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/gcc.cmake -DENABLE_STRICT=1 -DHAVE_DIAGNOSTIC=0
+      - func: "compile wiredtiger"
+        vars:
           posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/gcc.cmake -DGNU_C_VERSION=7 -DCMAKE_C_FLAGS="-ggdb" -DWITH_PIC=1
       - func: "compile wiredtiger"
         vars:
@@ -785,6 +788,9 @@ tasks:
       - func: "get project"
       - func: "compile wiredtiger"
         vars:
+          posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -DENABLE_STRICT=1 -DHAVE_DIAGNOSTIC=0
+      - func: "compile wiredtiger"
+        vars:
           posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/clang.cmake -DCLANG_C_VERSION=6.0 -DCLANG_CXX_VERSION=6.0 -DCMAKE_C_FLAGS="-ggdb" -DWITH_PIC=1
       - func: "compile wiredtiger"
         vars:
@@ -2680,6 +2686,60 @@ tasks:
     name: recovery-stress-test-3
     tags: ["stress-test-3", "stress-test-zseries-3"]
 
+  - name: format-stress-test-no-barrier
+    tags: ["stress-test-no-barrier"]
+    exec_timeout_secs: 25200
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger with builtins"
+        vars: 
+          NON_BARRIER_DIAGNOSTIC_YIELDS: -DNON_BARRIER_DIAGNOSTIC_YIELDS=1
+      - func: "format test script"
+        vars:
+          format_test_script_args: -e "SEGFAULT_SIGNALS=all" -b "catchsegv ./t" -t 360
+
+  - name: format-stress-sanitizer-test-no-barrier
+    tags: ["stress-test-no-barrier"]
+    exec_timeout_secs: 25200
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger address sanitizer"
+        vars: 
+          NON_BARRIER_DIAGNOSTIC_YIELDS: -DNON_BARRIER_DIAGNOSTIC_YIELDS=1
+      - func: "format test script"
+        vars:
+          format_test_script_args: -t 360
+          test_env_vars:
+            ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1"
+            ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer
+
+  - name: race-condition-stress-sanitizer-test-no-barrier
+    tags: ["stress-test-no-barrier"]
+    exec_timeout_secs: 25200
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger"
+        vars: 
+          posix_configure_flags: -DENABLE_STRICT=1 -DHAVE_DIAGNOSTIC=1 -DNON_BARRIER_DIAGNOSTIC_YIELDS=1 -DHAVE_BUILTIN_EXTENSION_LZ4=1 -DHAVE_BUILTIN_EXTENSION_SNAPPY=1 -DHAVE_BUILTIN_EXTENSION_ZLIB=1 -DHAVE_BUILTIN_EXTENSION_ZSTD=1
+      - func: "format test script"
+        vars:
+          format_test_script_args: -R -t 360
+          test_env_vars:
+            ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1"
+            ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v4/bin/llvm-symbolizer
+
+  - name: recovery-stress-test-no-barrier
+    tags: ["stress-test-no-barrier"]
+    exec_timeout_secs: 25200
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger"
+        vars:
+          posix_configure_flags: -DENABLE_STRICT=1 -DHAVE_DIAGNOSTIC=1 -DNON_BARRIER_DIAGNOSTIC_YIELDS=1 -DHAVE_BUILTIN_EXTENSION_LZ4=1 -DHAVE_BUILTIN_EXTENSION_SNAPPY=1 -DHAVE_BUILTIN_EXTENSION_ZLIB=1
+      - func: "recovery stress test script"
+        vars:
+          times: 25
+
   - name: format-abort-recovery-stress-test
     commands:
       # Allow 30 minutes beyond test runtime because recovery under load can cause the test to
@@ -3721,7 +3781,7 @@ buildvariants:
       -DCMAKE_BUILD_TYPE=UBSan
       -DCC_OPTIMIZE_LEVEL=-O1
       -DENABLE_STRICT=1
-      -DHAVE_DIAGNOTIC=1
+      -DHAVE_DIAGNOSTIC=1
       -DENABLE_STATIC=0
       -DENABLE_TCMALLOC=1
       -DCMAKE_PREFIX_PATH="$(pwd)/../TCMALLOC_LIB"
@@ -3802,6 +3862,7 @@ buildvariants:
     - name: ".stress-test-2"
     - name: ".stress-test-3"
     - name: ".stress-test-4"
+    - name: ".stress-test-no-barrier"
     - name: format-abort-recovery-stress-test
 
 - name: ubuntu2004-stress-tests-arm64
@@ -3840,6 +3901,7 @@ buildvariants:
     - name: ".stress-test-2"
     - name: ".stress-test-3"
     - name: ".stress-test-4"
+    - name: ".stress-test-no-barrier"
     - name: format-abort-recovery-stress-test
 
 - name: ubuntu2004-perf-tests