Import wiredtiger: 2fd676d2bca3a57b8cd704e980d7518d82015c0f from branch mongodb-4.4

ref: fb527a5411..2fd676d2bc
for: 4.3.1

WT-4976       Migrate Jenkins “wiredtiger-pull-request-builds” job to Evergreen
WT-4984       Migrate Jenkins “wiredtiger-clang-sanitizer” job to Evergreen
WT-4985       Migrate Jenkins “wiredtiger-sanitized-undefined” job to Evergreen
WT-4986       Migrate Jenkins “wiredtiger-linux-directio” job to Evergreen
WT-4993       Migrate Jenkins “wiredtiger-configure-combinations” job to Evergreen
WT-5114       Enable running WiredTiger with clang MemorySanitizer
WT-5157       Fix atomics usage in spinlock implementation
WT-5195       Reduce Python unit test build time for Evergreen Windows build variant
WT-5197       Fix sanitizer runtime flags in Evergreen

5 files changed, 244 insertions, 17 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index f7bbc66e996..0693b73721b 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -1,5 +1,5 @@
 {
-    "commit": "fb527a54117f3865aae8a25557849f1228d48205", 
+    "commit": "2fd676d2bca3a57b8cd704e980d7518d82015c0f", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-4.4"
diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h
index d65eea97b68..e5570f6f066 100644
--- a/src/third_party/wiredtiger/src/include/mutex.h
+++ b/src/third_party/wiredtiger/src/include/mutex.h
@@ -99,7 +99,8 @@ struct __wt_rwlock { /* Read/write lock */
 struct __wt_spinlock {
 #if SPINLOCK_TYPE == SPINLOCK_GCC
     WT_CACHE_LINE_PAD_BEGIN
-    volatile int lock;
+    volatile bool lock;
+    uint8_t unused[7];
 #elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX || \
   SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE || SPINLOCK_TYPE == SPINLOCK_MSVC
     wt_mutex_t lock;
diff --git a/src/third_party/wiredtiger/src/include/mutex.i b/src/third_party/wiredtiger/src/include/mutex.i
index 45a0b3ab0f0..c520014c053 100644
--- a/src/third_party/wiredtiger/src/include/mutex.i
+++ b/src/third_party/wiredtiger/src/include/mutex.i
@@ -67,7 +67,7 @@ __wt_spin_trylock(WT_SESSION_IMPL *session, WT_SPINLOCK *t)
 {
     WT_UNUSED(session);
 
-    return (__atomic_test_and_set(&t->lock, __ATOMIC_ACQUIRE) ? 0 : EBUSY);
+    return (!__atomic_test_and_set(&t->lock, __ATOMIC_ACQUIRE) ? 0 : EBUSY);
 }
 
 /*
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index 40dfc5d8874..d1030c51c0a 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -82,6 +82,15 @@ functions:
         set -o verbose
 
         ${test_env_vars|} ${make_command|make} VERBOSE=1 check -C ${directory}  ${smp_command|} 2>&1
+  "make check all":
+    command: shell.exec
+    params:
+      working_dir: "wiredtiger/build_posix"
+      script: |
+        set -o errexit
+        set -o verbose
+
+        ${test_env_vars|} ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1
   "upload artifact":
     - command: archive.targz_pack
       params:
@@ -113,7 +122,7 @@ post:
   - func: "cleanup"
 
 tasks:
-## Base compile task on posix flavours
+  # Base compile task on posix flavours
   - name: compile
     tags: ["pull_request"]
     commands:
@@ -126,23 +135,80 @@ tasks:
       - func: "get project"
       - func: "compile wiredtiger"
         vars:
-          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer CFLAGS="-fsanitize=address -ggdb"
+          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb"
           posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static
 
+  - name: compile-msan
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger"
+        vars:
+          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=memory -ggdb"
+          posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static
+
+  - name: compile-ubsan
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger"
+        vars:
+          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ CFLAGS="-fsanitize=undefined -ggdb"
+          posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic
+
+  # production build with --disable-shared
+  - name: compile-production-disable-shared
+    tags: ["pull_request"]
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger"
+        vars:
+          posix_configure_flags: --enable-silent-rules --enable-strict --disable-shared
+
+  # production build with --disable-static
+  - name: compile-production-disable-static
+    tags: ["pull_request"]
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger"
+        vars:
+          posix_configure_flags: --enable-silent-rules --enable-strict --disable-static --enable-lz4 --enable-snappy --enable-zlib --enable-zstd --enable-python
+
   - name: make-check-test
     depends_on:
       - name: compile
     commands:
       - func: "fetch artifacts"
       - func: "compile wiredtiger"
-      - command: shell.exec
-        params:
-          working_dir: "wiredtiger/build_posix"
-          script: |
-            set -o errexit
-            set -o verbose
+      - func: "make check all"
 
-            ${test_env_vars|} ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1
+  - name: make-check-msan-test
+    depends_on:
+      - name: compile-msan
+    commands:
+      - func: "fetch artifacts"
+        vars:
+          dependent_task: compile-msan
+      - func: "compile wiredtiger"
+        vars:
+          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=memory -ggdb"
+          posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static
+      - func: "make check all"
+        vars:
+          test_env_vars: MSAN_OPTIONS=abort_on_error=1:disable_coredump=0 MSAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer
+
+  - name: make-check-asan-test
+    depends_on:
+      - name: compile-asan
+    commands:
+      - func: "fetch artifacts"
+        vars:
+          dependent_task: compile-asan
+      - func: "compile wiredtiger"
+        vars:
+          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=address -fno-omit-frame-pointer -ggdb"
+          posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static
+      - func: "make check all"
+        vars:
+          test_env_vars: ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer
 
   # Start of normal make check test tasks
 
@@ -178,10 +244,41 @@ tasks:
           dependent_task: compile-asan
       - func: "compile wiredtiger"
         vars:
-          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer CFLAGS="-fsanitize=address -ggdb"
+          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ CFLAGS="-fsanitize=address -ggdb"
           posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic --disable-static
       - func: "make check directory"
         vars:
+          test_env_vars: ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer
+          directory: examples/c
+
+  - name: examples-c-production-disable-shared-test
+    tags: ["pull_request"]
+    depends_on:
+      - name: compile-production-disable-shared
+    commands:
+      - func: "fetch artifacts"
+        vars:
+          dependent_task: compile-production-disable-shared
+      - func: "compile wiredtiger"
+        vars:
+          posix_configure_flags: --enable-silent-rules --enable-strict --disable-shared
+      - func: "make check directory"
+        vars:
+          directory: examples/c
+
+  - name: examples-c-production-disable-static-test
+    tags: ["pull_request"]
+    depends_on:
+      - name: compile-production-disable-static
+    commands:
+      - func: "fetch artifacts"
+        vars:
+          dependent_task: compile-production-disable-static
+      - func: "compile wiredtiger"
+        vars:
+          posix_configure_flags: --enable-silent-rules --enable-strict --disable-static --enable-lz4 --enable-snappy --enable-zlib --enable-zstd --enable-python
+      - func: "make check directory"
+        vars:
           directory: examples/c
 
   - name: bloom-test
@@ -318,6 +415,23 @@ tasks:
 
   # End of normal make check test tasks
 
+  - name: ubsan-test
+    depends_on:
+      - name: compile-ubsan
+    commands:
+      - func: "fetch artifacts"
+      - func: "compile wiredtiger"
+        vars:
+          configure_env_vars: CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ CFLAGS="-fsanitize=undefined -ggdb"
+          posix_configure_flags: --enable-silent-rules --enable-strict --enable-diagnostic
+      - command: shell.exec
+        params:
+          working_dir: "wiredtiger/build_posix/examples/c"
+          script: |
+            set -o errexit
+            set -o verbose
+            UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1:abort_on_error=1:disable_coredump=0 ./ex_access
+
   # Start of csuite test tasks
 
   - name: csuite-import-test
@@ -888,7 +1002,8 @@ tasks:
             set -o errexit
             set -o verbose
 
-            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [c] -v 2 ${smp_command|} 2>&1
+            # Reserve this bucket only for compat tests, which take a long time to run
+            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py compat -v 2 ${smp_command|} 2>&1
 
   - name: unit-test-bucket02
     tags: ["pull_request", "unit_test"]
@@ -903,7 +1018,9 @@ tasks:
             set -o errexit
             set -o verbose
 
-            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [defg] -v 2 ${smp_command|} 2>&1
+            # Non-compat tests in the 'c' family
+            non_compat_tests=$(ls ../test/suite/test_c*.py | xargs -n1 basename | grep -v compat)
+            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_compat_tests -v 2 ${smp_command|} 2>&1
 
   - name: unit-test-bucket03
     tags: ["pull_request", "unit_test"]
@@ -918,7 +1035,7 @@ tasks:
             set -o errexit
             set -o verbose
 
-            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [hijk] -v 2 ${smp_command|} 2>&1
+            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [defghijk] -v 2 ${smp_command|} 2>&1
 
   - name: unit-test-bucket04
     tags: ["pull_request", "unit_test"]
@@ -963,7 +1080,8 @@ tasks:
             set -o errexit
             set -o verbose
 
-            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [t] -v 2 ${smp_command|} 2>&1
+            # Reserve this bucket only for timestamp tests, which take a long time to run
+            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py timestamp -v 2 ${smp_command|} 2>&1
 
   - name: unit-test-bucket07
     tags: ["pull_request", "unit_test"]
@@ -978,6 +1096,9 @@ tasks:
             set -o errexit
             set -o verbose
 
+            # Non-timestamp tests in the 't' family
+            non_ts_tests=$(ls ../test/suite/test_t*.py | xargs -n1 basename | grep -v timestamp)
+            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py $non_ts_tests -v 2 ${smp_command|} 2>&1
             ${test_env_vars|} ${python_binary|python} ../test/suite/run.py [uvwxyz] -v 2 ${smp_command|} 2>&1
 
   # End of Python unit test tasks
@@ -1236,6 +1357,33 @@ tasks:
             set -o verbose
             sh dist/s_clang-scan 2>&1
 
+  - name: configure-combinations
+    commands:
+      - func: "get project"
+      - command: shell.exec
+        params:
+          working_dir: "wiredtiger/test/evergreen"
+          script: |
+            set -o errexit
+            set -o verbose
+            ./configure_combinations.sh ${smp_command|} 2>&1
+  
+  - name: linux-directio
+    depends_on:
+    - name: compile
+    commands:
+      - func: "fetch artifacts"
+      - func: "compile wiredtiger"
+      - command: shell.exec
+        params:
+          working_dir: "wiredtiger/build_posix/test/format"
+          script: |
+            set -o errexit
+            set -o verbose
+            for i in {1..3}; do
+              ./t -1 -C "direct_io=[data]" -c ../../../test/format/CONFIG.stress
+            done
+
 buildvariants:
 - name: ubuntu1804
   display_name: Ubuntu 18.04
@@ -1248,6 +1396,13 @@ buildvariants:
     make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make
   tasks:
     - name: ".pull_request !.windows_only"
+    - name: compile-msan
+    - name: make-check-msan-test
+    - name: compile-ubsan
+    - name: ubsan-test
+    - name: linux-directio
+    - name: make-check-asan-test
+    - name: configure-combinations
 
 - name: ubuntu1804-python3
   display_name: Ubuntu 18.04 (Python3)
diff --git a/src/third_party/wiredtiger/test/evergreen/configure_combinations.sh b/src/third_party/wiredtiger/test/evergreen/configure_combinations.sh
new file mode 100755
index 00000000000..441c7d3b12a
--- /dev/null
+++ b/src/third_party/wiredtiger/test/evergreen/configure_combinations.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+cd $(git rev-parse --show-toplevel)
+echo `pwd`
+sh build_posix/reconf
+
+curdir=`pwd`
+
+flags="CFLAGS=\"-Werror -Wall -Wextra -Waddress -Waggregate-return -Wbad-function-cast -Wcast-align -Wdeclaration-after-statement -Wformat-security -Wformat-nonliteral -Wformat=2 -Wmissing-declarations -Wmissing-field-initializers -Wmissing-prototypes -Wnested-externs -Wno-unused-parameter -Wpointer-arith -Wredundant-decls -Wshadow -Wundef -Wunused -Wwrite-strings -O -fno-strict-aliasing -Wuninitialized\"
+CC=clang CFLAGS=\"-Wall -Werror -Qunused-arguments -Wno-self-assign -Wno-parentheses-equality -Wno-array-bounds\""
+
+options="--enable-diagnostic
+--disable-shared
+--disable-static
+--enable-java --enable-python
+--enable-snappy --enable-zlib --enable-lz4
+--with-builtins=lz4,snappy,zlib
+--enable-diagnostic --enable-java --enable-python"
+
+saved_IFS=$IFS
+cr_IFS="
+"
+
+# This function may alter the current directory on failure
+BuildTest() {
+        extra_config=--enable-silent-rules
+        echo "Building: $1, $2"
+        rm -rf ./build || return 1
+        mkdir build || return 1
+        cd ./build
+        eval ../configure $extra_config "$1" "$2" \
+                 --prefix="$insdir" || return 1
+        eval make "$3" || return 1
+        make -C examples/c check VERBOSE=1 > /dev/null || return 1
+        case "$2" in
+                # Skip the install step with Python.  Even with --prefix, the
+                # install tries to write to /usr/lib64/python2.7/site-packages .
+                *enable-python* )  doinstall=false;;
+                # Non-shared doesn't yet work: library is not found at link step (??)
+                *disable-shared* ) doinstall=false;;
+                * )                doinstall=true;;
+        esac
+        if $doinstall; then
+                eval make install || return 1
+                cflags=`pkg-config wiredtiger --cflags --libs`
+                [ "$1"  == "CC=clang" ] && compiler="clang" || compiler="cc"
+                echo $compiler -o ./smoke ../examples/c/ex_smoke.c $cflags
+                $compiler -o ./smoke ../examples/c/ex_smoke.c  $cflags|| return 1
+                LD_LIBRARY_PATH=$insdir/lib ./smoke || return 1
+        fi
+        return 0
+}
+
+ecode=0
+insdir=`pwd`/installed
+export PKG_CONFIG_PATH=$insdir/lib/pkgconfig
+IFS="$cr_IFS"
+for flag in $flags ; do
+        for option in $options ; do
+               cd "$curdir"
+               IFS="$saved_IFS"
+               if ! BuildTest "$flag" "$option" "$@"; then
+                       ecode=1
+                       echo "*** ERROR: $flag, $option"
+               fi
+               IFS="$cr_IFS"
+               echo hello
+       done
+done
+IFS=$saved_IFS
+exit $ecode