Import wiredtiger: 3a42ced49595b1b082263047cfd2e4d536d62514 from branch mongodb-4.4

ref: ddace22cbd..3a42ced495
for: 4.3.1

WT-4987       Migrate Jenkins “wiredtiger-test-checkpoint” job to Evergreen
WT-5030       Migrate Jenkins “wiredtiger-linux-no-ftruncate” job to Evergreen
WT-5031       Migrate Jenkins “wiredtiger-nightly-package” job to Evergreen
WT-5032       Migrate Jenkins “wiredtiger-syscall-linux” job to Evergreen
WT-5106       Remove temporary files in clang-format script
WT-5189       Python2: handle errors that occur under the concurrent test option (-j)
WT-5218       Improve eviction to differentiate between clean and dirty pages with WT_CACHE_EVICT_NOKEEP readgen

5 files changed, 211 insertions, 45 deletions

diff --git a/src/third_party/wiredtiger/dist/s_clang-format b/src/third_party/wiredtiger/dist/s_clang-format
index 58e4b59f077..1673946ee08 100755
--- a/src/third_party/wiredtiger/dist/s_clang-format
+++ b/src/third_party/wiredtiger/dist/s_clang-format
@@ -2,6 +2,9 @@
 
 set -o pipefail
 
+t=__wt.$$
+trap 'rm -rf $t' 0 1 2 3 13 15
+
 download_clang_format() {
 	if [ `uname` = "Linux" ]; then
 		curl https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-3.8-rhel55.tar.gz -o dist/clang-format.tar.gz
@@ -45,16 +48,15 @@ esac
 # Don't format inplace with -i flag.
 # We want to be able to detect modifications.
 for f in $search; do
-	tempfile=$(mktemp)
 	cat "$f" | \
             clang-format --fallback-style=none | \
             python dist/s_goto.py | \
-            python dist/s_comment.py > "$tempfile" || exit 1
-	cmp --silent "$f" "$tempfile"
+            python dist/s_comment.py > "$t" || exit 1
+	cmp --silent "$f" "$t"
 	if test $? -ne 0; then
 		if test $# -eq 0 ; then
 			echo "Modifying $f"
 		fi
-		mv "$tempfile" "$f"
+		cp "$t" "$f"
 	fi
 done
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 1cc649c677d..4357f825c77 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -1,5 +1,5 @@
 {
-    "commit": "ddace22cbd344d440b523e2b3158b1d720cc4de9", 
+    "commit": "3a42ced49595b1b082263047cfd2e4d536d62514", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-4.4"
diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i
index 69bb0dec90a..550ec45fbf0 100644
--- a/src/third_party/wiredtiger/src/include/btree.i
+++ b/src/third_party/wiredtiger/src/include/btree.i
@@ -489,6 +489,12 @@ __wt_page_only_modify_set(WT_SESSION_IMPL *session, WT_PAGE *page)
     if (page->modify->page_state < WT_PAGE_DIRTY &&
       __wt_atomic_add32(&page->modify->page_state, 1) == WT_PAGE_DIRTY_FIRST) {
         __wt_cache_dirty_incr(session, page);
+        /*
+         * In the event we dirty a page which is flagged for eviction soon, we update its read
+         * generation to avoid evicting a dirty page prematurely.
+         */
+        if (page->read_gen == WT_READGEN_WONT_NEED)
+            __wt_cache_read_gen_new(session, page);
 
         /*
          * We won the race to dirty the page, but another thread could have committed in the
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index d1030c51c0a..fd5b9bf4b3e 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -9,13 +9,13 @@ functions:
     params:
       directory: wiredtiger
   "fetch artifacts" :
-    - command: s3.get
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${dependent_task|compile}_${build_id}.tgz
-        bucket: build_external
-        extract_to: wiredtiger
+    command: s3.get
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${dependent_task|compile}_${build_id}.tgz
+      bucket: build_external
+      extract_to: wiredtiger
   "fetch artifacts from little-endian" :
     - command: s3.get
       params:
@@ -45,7 +45,21 @@ functions:
     params:
       script: |
         git clone https://github.com/wiredtiger/mongo-tests
-  "compile wiredtiger":
+  "configure wiredtiger": &configure_wiredtiger
+    command: shell.exec
+    params:
+      working_dir: "wiredtiger/build_posix"
+      shell: bash
+      script: |
+        set -o errexit
+        set -o verbose
+        if [ "$OS" != "Windows_NT" ]; then
+          sh reconf
+          ${configure_env_vars|CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb"} \
+            ../configure ${configure_python_setting|} \
+            ${posix_configure_flags|--enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL}
+        fi
+  "make wiredtiger": &make_wiredtiger
     command: shell.exec
     params:
       working_dir: "wiredtiger"
@@ -53,15 +67,11 @@ functions:
       script: |
         set -o errexit
         set -o verbose
-        if [ "Windows_NT" = "$OS" ]; then
+        if [ "Windows_NT" == "$OS" ]; then
           pip install scons==3.1.1
           scons-3.1.1.bat ${win_configure_flags|--enable-python=c:\\swigwin-3.0.2\\swig.exe --enable-diagnostic} ${smp_command|}
         else
           cd build_posix
-          sh ./reconf
-          ${configure_env_vars|CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb"} \
-            ../configure ${configure_python_setting|} \
-            ${posix_configure_flags|--enable-silent-rules --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-static --prefix=$(pwd)/LOCAL_INSTALL}
           ${make_command|make} ${smp_command|} 2>&1
 
           # On macOS, change the binary location with install_name_tool since DYLD_LIBRARY_PATH
@@ -73,6 +83,20 @@ functions:
             install_name_tool -change /usr/local/lib/libwiredtiger-$WT_VERSION.dylib $(pwd)/.libs/libwiredtiger-$WT_VERSION.dylib .libs/wt
           fi
         fi
+  "compile wiredtiger":
+    - *configure_wiredtiger
+    - *make_wiredtiger
+  "compile wiredtiger no linux ftruncate":
+    - *configure_wiredtiger
+    - command: shell.exec
+      params:
+        working_dir: "wiredtiger/build_posix"
+        shell: bash
+        script: |
+          set -o errexit
+          set -o verbose
+          echo '#undef HAVE_FTRUNCATE' >> wiredtiger_config.h
+    - *make_wiredtiger
   "make check directory":
     command: shell.exec
     params:
@@ -91,6 +115,25 @@ functions:
         set -o verbose
 
         ${test_env_vars|} ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1
+  "unit test":
+    command: shell.exec
+    params:
+      working_dir: "wiredtiger/build_posix"
+      script: |
+        set -o errexit
+        set -o verbose
+
+        ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -v 2 ${smp_command|} 2>&1
+  "test format":
+    command: shell.exec
+    params:
+      working_dir: "wiredtiger/build_posix/test/format"
+      script: |
+        set -o errexit
+        set -o verbose
+        for i in $(seq ${times|1}); do
+          ./t ${test_format_args|-1 -c ../../../test/format/CONFIG.stress}
+        done
   "upload artifact":
     - command: archive.targz_pack
       params:
@@ -115,6 +158,15 @@ functions:
         rm -rf "wiredtiger"
         rm -rf "wiredtiger.tgz"
 
+  "checkpoint test":
+    command: shell.exec
+    params:
+      working_dir: "wiredtiger/build_posix/test/checkpoint"
+      script: |
+        set -o errexit
+        set -o verbose
+        ./t ${checkpoint_args} 2>&1
+
 pre:
   - func: "cleanup"
 post:
@@ -172,6 +224,13 @@ tasks:
         vars:
           posix_configure_flags: --enable-silent-rules --enable-strict --disable-static --enable-lz4 --enable-snappy --enable-zlib --enable-zstd --enable-python
 
+  - name: compile-linux-no-ftruncate
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger no linux ftruncate"
+        vars:
+          posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-python
+
   - name: make-check-test
     depends_on:
       - name: compile
@@ -210,6 +269,18 @@ tasks:
         vars:
           test_env_vars: ASAN_OPTIONS=detect_leaks=1:abort_on_error=1:disable_coredump=0 ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer
 
+  - name: make-check-linux-no-ftruncate-test
+    depends_on:
+      - name: compile-linux-no-ftruncate
+    commands:
+      - func: "fetch artifacts"
+        vars:
+          dependent_task: compile-linux-no-ftruncate
+      - func: "compile wiredtiger no linux ftruncate"
+        vars:
+          posix_configure_flags: --enable-silent-rules --enable-diagnostic --enable-strict --enable-python
+      - func: "make check all"
+
   # Start of normal make check test tasks
 
   - name: lang-python-test
@@ -953,19 +1024,16 @@ tasks:
     - name: compile
     commands:
       - func: "fetch artifacts"
-      - command: shell.exec
-        params:
-          working_dir: "wiredtiger/build_posix"
-          script: |
-            set -o errexit
-            set -o verbose
+      - func: "unit test"
 
-            # Only Windows and OS X variants are expected to run this task
-            #
-            # Avoid /usr/bin/python, at least on macOS: with System Integrity
-            # Protection enabled, it ignores DYLD_LIBRARY_PATH and hence
-            # doesn't find the WiredTiger library in the local tree.
-            ${test_env_vars|} ${python_binary|python} ../test/suite/run.py -v 2 ${smp_command|} 2>&1
+  - name: unit-linux-no-ftruncate-test
+    depends_on:
+    - name: compile-linux-no-ftruncate
+    commands:
+      - func: "fetch artifacts"
+        vars:
+          dependent_task: compile-linux-no-ftruncate
+      - func: "unit test"
 
   # Break out Python unit tests into multiple buckets/tasks based on test name and runtime
   # The test/suite/run.py script can work out test names by casting each command argument
@@ -1229,19 +1297,19 @@ tasks:
     commands:
       - func: "fetch artifacts"
       - func: "compile wiredtiger"
+      - func: "test format"
+        vars:
+          times: 10
+          test_format_args: -1 -h "WT_TEST.$i" -c ../../../test/format/CONFIG.endian
       - command: shell.exec
         params:
           working_dir: "wiredtiger/build_posix/test/format"
+          shell: bash
           script: |
             set -o errexit
             set -o verbose
-            for i in $(seq 10)
-            do
-              ./t -1 -h "WT_TEST.$i" -c $(pwd)/../../../test/format/CONFIG.endian
-            done
             # Archive the WT_TEST directories which include the generated wt data files
             tar -zcvf WT_TEST.tgz WT_TEST*
-          shell: bash
       - command: s3.put
         params:
           aws_secret: ${aws_secret}
@@ -1290,19 +1358,19 @@ tasks:
     commands:
       - func: "fetch artifacts"
       - func: "compile wiredtiger"
+      - func: "test format"
+        vars:
+          times: 10
+          test_format_args: -1 -h "WT_TEST.$i" -c ../../../test/format/CONFIG.endian
       - command: shell.exec
         params:
           working_dir: "wiredtiger/build_posix/test/format"
+          shell: bash
           script: |
             set -o errexit
             set -o verbose
-            for i in $(seq 10)
-            do
-              ./t -1 -h "WT_TEST.$i" -c $(pwd)/../../../test/format/CONFIG.endian
-            done
             # Archive the WT_TEST directories which include the generated wt data files
             tar -zcvf WT_TEST.tgz WT_TEST*
-          shell: bash
       - command: s3.put
         params:
           aws_secret: ${aws_secret}
@@ -1374,15 +1442,64 @@ tasks:
     commands:
       - func: "fetch artifacts"
       - func: "compile wiredtiger"
+      - func: "test format"
+        vars:
+          times: 3
+          test_format_args: -1 -C "direct_io=[data]" -c ../../../test/format/CONFIG.stress
+
+  - name: format-linux-no-ftruncate
+    depends_on:
+    - name: compile-linux-no-ftruncate
+    commands:
+      - func: "fetch artifacts"
+        vars:
+          dependent_task: compile-linux-no-ftruncate
+      - func: "compile wiredtiger no linux ftruncate"
+      - func: "test format"
+        vars:
+          times: 3
+
+  - name: package
+    commands:
+      - func: "get project"
+      - command: shell.exec
+        params:
+          working_dir: "wiredtiger/dist"
+          script: |
+            set -o errexit
+            set -o verbose
+            env CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:/opt/java/jdk11/bin:$PATH sh s_release `date +%Y%m%d`
+
+  - name: syscall-linux
+    depends_on:
+    - name: compile
+    commands:
+      - func: "fetch artifacts"
+      - func: "compile wiredtiger"
       - command: shell.exec
         params:
-          working_dir: "wiredtiger/build_posix/test/format"
+          working_dir: "wiredtiger/test/syscall"
           script: |
             set -o errexit
             set -o verbose
-            for i in {1..3}; do
-              ./t -1 -C "direct_io=[data]" -c ../../../test/format/CONFIG.stress
-            done
+            ${python_binary|python} syscall.py --verbose
+
+  - name: checkpoint-filetypes-test
+    commands:
+      - func: "get project"
+      - func: "compile wiredtiger"
+        vars:
+          # Don't use diagnostic - this test looks for timing problems that are more likely to occur without it
+          posix_configure_flags: --enable-strict
+      - func: "checkpoint test"
+        vars:
+          checkpoint_args: -t m -n 1000000 -k 5000000 -C cache_size=100MB
+      - func: "checkpoint test"
+        vars:
+          checkpoint_args: -t r -n 1000000 -k 5000000 -C cache_size=100MB
+      - func: "checkpoint test"
+        vars:
+          checkpoint_args: -t c -n 1000000 -k 5000000 -C cache_size=100MB
 
 buildvariants:
 - name: ubuntu1804
@@ -1403,6 +1520,7 @@ buildvariants:
     - name: linux-directio
     - name: make-check-asan-test
     - name: configure-combinations
+    - name: checkpoint-filetypes-test
 
 - name: ubuntu1804-python3
   display_name: Ubuntu 18.04 (Python3)
@@ -1420,6 +1538,42 @@ buildvariants:
     - name: ".unit_test"
     - name: conf-dump-test
 
+- name: package
+  display_name: Package
+  batchtime: 1440 # 1 day
+  run_on:
+  - ubuntu1804-test
+  tasks:
+    - name: package
+
+- name: syscall-linux
+  display_name: Syscall Linux
+  batchtime: 10080 # a week
+  run_on:
+  - ubuntu1804-test
+  expansions:
+    test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd)
+    smp_command: -j $(grep -c ^processor /proc/cpuinfo)
+    make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make
+  tasks:
+    - name: compile
+    - name: syscall-linux
+
+- name: linux-no-ftruncate
+  display_name: Linux no ftruncate
+  batchtime: 10080 # 1 day
+  run_on:
+  - ubuntu1804-test
+  expansions:
+    test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs top_srcdir=$(pwd)/.. top_builddir=$(pwd)
+    smp_command: -j $(grep -c ^processor /proc/cpuinfo)
+    make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make
+  tasks:
+    - name: compile-linux-no-ftruncate
+    - name: make-check-linux-no-ftruncate-test
+    - name: unit-linux-no-ftruncate-test
+    - name: format-linux-no-ftruncate
+
 - name: rhel80
   display_name: RHEL 8.0
   run_on:
diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py
index b2fec97cb17..5e2722cfdd6 100755
--- a/src/third_party/wiredtiger/test/suite/wttest.py
+++ b/src/third_party/wiredtiger/test/suite/wttest.py
@@ -411,6 +411,8 @@ class WiredTigerTestCase(unittest.TestCase):
         # This approach works for all our support Python versions and
         # is suggested by one of the answers in:
         # https://stackoverflow.com/questions/4414234/getting-pythons-unittest-results-in-a-teardown-method
+        # In addition, check to make sure exc_info is "clean", because
+        # the ConcurrencyTestSuite in Python2 indicates failures using that.
         if hasattr(self, '_outcome'):  # Python 3.4+
             result = self.defaultTestResult()  # these 2 methods have no side effects
             self._feedErrorsToResult(result, self._outcome.errors)
@@ -418,7 +420,9 @@ class WiredTigerTestCase(unittest.TestCase):
             result = getattr(self, '_outcomeForDoCleanups', self._resultForDoCleanups)
         error = self.list2reason(result, 'errors')
         failure = self.list2reason(result, 'failures')
-        passed = not error and not failure
+        exc_failure = (sys.exc_info() != (None, None, None))
+
+        passed = not error and not failure and not exc_failure
 
         self.pr('finishing')