Import wiredtiger: be699d75cf07a0383912a7e7031a8056c4e0be12 from branch mongodb-5.0

ref: 8e6373247b..be699d75cf
for: 5.1.0

WT-6555       Fix memory error in test_txn13
WT-7348       Complete CMake POSIX support
WT-7504       Fix test_hs21 cache stuck dirty
WT-7567       Rework tiered storage reconfigure
WT-7569       Fix wrongly squash an out of order timestamp update

Reverted ticket(s):
WT-7531       Consider update restore eviction as a progress

15 files changed, 153 insertions, 65 deletions

diff --git a/src/third_party/wiredtiger/build_cmake/configs/modes.cmake b/src/third_party/wiredtiger/build_cmake/configs/modes.cmake
index 5be475c51f1..29b5672da6d 100644
--- a/src/third_party/wiredtiger/build_cmake/configs/modes.cmake
+++ b/src/third_party/wiredtiger/build_cmake/configs/modes.cmake
@@ -9,6 +9,15 @@
 # Establishes build configuration modes we can use when compiling.
 
 # Create an ASAN build variant
+
+# Clang and GCC have slightly different linker names for the ASAN library.
+set(libasan)
+if("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang")
+    set(libasan "-static-libsan")
+else()
+    set(libasan "-static-libasan")
+endif()
+
 set(CMAKE_C_FLAGS_ASAN
     "${CMAKE_C_FLAGS_DEBUG} -fsanitize=address -fno-omit-frame-pointer" CACHE STRING
     "Flags used by the C compiler for ASan build type or configuration." FORCE)
@@ -18,11 +27,11 @@ set(CMAKE_CXX_FLAGS_ASAN
     "Flags used by the C++ compiler for ASan build type or configuration." FORCE)
 
 set(CMAKE_EXE_LINKER_FLAGS_ASAN
-    "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=address -static-libasan" CACHE STRING
+    "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=address ${libasan}" CACHE STRING
     "Linker flags to be used to create executables for ASan build type." FORCE)
 
 set(CMAKE_SHARED_LINKER_FLAGS_ASAN
-    "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=address -static-libasan" CACHE STRING
+    "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=address ${libasan}" CACHE STRING
     "Linker lags to be used to create shared libraries for ASan build type." FORCE)
 
 mark_as_advanced(
@@ -33,6 +42,13 @@ mark_as_advanced(
 )
 
 # Create an UBSAN build variant
+
+# Clang doesn't need to link ubsan, this is only a GCC requirement.
+set(libubsan "")
+if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+    set(libubsan "-lubsan")
+endif()
+
 set(CMAKE_C_FLAGS_UBSAN
     "${CMAKE_C_FLAGS_DEBUG} -fsanitize=undefined -fno-omit-frame-pointer" CACHE STRING
     "Flags used by the C compiler for UBSan build type or configuration." FORCE)
@@ -42,11 +58,11 @@ set(CMAKE_CXX_FLAGS_UBSAN
     "Flags used by the C++ compiler for UBSan build type or configuration." FORCE)
 
 set(CMAKE_EXE_LINKER_FLAGS_UBSAN
-    "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=undefined -lubsan" CACHE STRING
+    "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=undefined ${libubsan}" CACHE STRING
     "Linker flags to be used to create executables for UBSan build type." FORCE)
 
 set(CMAKE_SHARED_LINKER_FLAGS_UBSAN
-    "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=undefined -lubsan" CACHE STRING
+    "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -fsanitize=undefined ${libubsan}" CACHE STRING
     "Linker lags to be used to create shared libraries for UBSan build type." FORCE)
 
 mark_as_advanced(
diff --git a/src/third_party/wiredtiger/build_cmake/install/install.cmake b/src/third_party/wiredtiger/build_cmake/install/install.cmake
index 8573b10ea1d..ec9b30234d6 100644
--- a/src/third_party/wiredtiger/build_cmake/install/install.cmake
+++ b/src/third_party/wiredtiger/build_cmake/install/install.cmake
@@ -35,32 +35,3 @@ if(WT_POSIX)
         DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
     )
 endif()
-
-# Install our wiredtiger compressor extensions (provided we have enabled/built them).
-if(HAVE_BUILTIN_EXTENSION_LZ4 OR ENABLE_LZ4)
-    install(TARGETS wiredtiger_lz4
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    )
-endif()
-
-if(HAVE_BUILTIN_EXTENSION_SNAPPY OR ENABLE_SNAPPY)
-    install(TARGETS wiredtiger_snappy
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    )
-endif()
-
-if(HAVE_BUILTIN_EXTENSION_ZLIB OR ENABLE_ZLIB)
-    install(TARGETS wiredtiger_zlib
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    )
-endif()
-
-if(HAVE_BUILTIN_EXTENSION_ZSTD OR ENABLE_ZSTD)
-    install(TARGETS wiredtiger_zstd
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    )
-endif()
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 9464c079a8f..a5e5d6b9acc 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-5.0",
-    "commit": "8e6373247bd61fd6029b9c80599bea3ebe3b1262"
+    "commit": "be699d75cf07a0383912a7e7031a8056c4e0be12"
 }
diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c
index 57de379f09e..6fc62f0a52b 100644
--- a/src/third_party/wiredtiger/src/btree/bt_split.c
+++ b/src/third_party/wiredtiger/src/btree/bt_split.c
@@ -2257,12 +2257,10 @@ __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, WT_MULTI *multi)
      * Pages with unresolved changes are not marked clean during reconciliation, do it now.
      *
      * Don't count this as eviction making progress, we did a one-for-one rewrite of a page in
-     * memory, typical in the case of cache pressure unless the cache is configured for scrub and
-     * page doesn't have any skipped updates.
+     * memory, typical in the case of cache pressure.
      */
     __wt_page_modify_clear(session, page);
-    if (F_ISSET(S2C(session)->cache, WT_CACHE_EVICT_SCRUB) && !multi->supd_restore)
-        F_SET_ATOMIC(page, WT_PAGE_EVICT_NO_PROGRESS);
+    F_SET_ATOMIC(page, WT_PAGE_EVICT_NO_PROGRESS);
     __wt_ref_out(session, ref);
 
     /* Swap the new page into place. */
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index 3d7b5fed416..526fdbfba86 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -1150,6 +1150,7 @@ err:
      * the sweep server.
      */
     WT_TRET(__wt_sweep_destroy(session));
+    WT_TRET(__wt_tiered_storage_destroy(session));
 
     /*
      * Shut down the checkpoint and capacity server threads: we don't want to throttle writes and
diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c
index 20467b4228b..b86ca4eb616 100644
--- a/src/third_party/wiredtiger/src/conn/conn_open.c
+++ b/src/third_party/wiredtiger/src/conn/conn_open.c
@@ -206,7 +206,7 @@ __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[])
      * can know if statistics are enabled or not.
      */
     WT_RET(__wt_statlog_create(session, cfg));
-    WT_RET(__wt_tiered_storage_create(session, cfg, false));
+    WT_RET(__wt_tiered_storage_create(session, cfg));
     WT_RET(__wt_logmgr_create(session));
 
     /*
diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c
index 8881d3783ef..c8d1330d7ff 100644
--- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c
+++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c
@@ -426,7 +426,7 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg)
     WT_ERR(__wt_logmgr_reconfig(session, cfg));
     WT_ERR(__wt_lsm_manager_reconfig(session, cfg));
     WT_ERR(__wt_statlog_create(session, cfg));
-    WT_ERR(__wt_tiered_storage_create(session, cfg, true));
+    WT_ERR(__wt_tiered_conn_config(session, cfg, true));
     WT_ERR(__wt_sweep_config(session, cfg));
     WT_ERR(__wt_timing_stress_config(session, cfg));
     WT_ERR(__wt_verbose_config(session, cfg));
diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c
index 25871cce3c0..24a4a5a16f7 100644
--- a/src/third_party/wiredtiger/src/conn/conn_tiered.c
+++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c
@@ -495,7 +495,7 @@ __tiered_mgr_start(WT_CONNECTION_IMPL *conn)
  *     Start the tiered storage subsystem.
  */
 int
-__wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig)
+__wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[])
 {
     WT_CONNECTION_IMPL *conn;
     WT_DECL_RET;
@@ -504,10 +504,6 @@ __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[], bool rec
     conn = S2C(session);
     start = false;
 
-    /* Destroy any existing thread since we could be a reconfigure. */
-    WT_RET(__wt_tiered_storage_destroy(session));
-    if (reconfig)
-        WT_RET(__wt_tiered_conn_config(session, cfg, reconfig));
     WT_RET(__tiered_manager_config(session, cfg, &start));
 
     /* Start the internal thread. */
@@ -516,7 +512,6 @@ __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[], bool rec
 
     WT_ERR(__wt_open_internal_session(conn, "storage-server", true, 0, 0, &conn->tiered_session));
     session = conn->tiered_session;
-    WT_ERR(__wt_txn_reconfigure(session, "isolation=read-uncommitted"));
 
     /* Start the thread. */
     WT_ERR(__wt_thread_create(session, &conn->tiered_tid, __tiered_server, session));
diff --git a/src/third_party/wiredtiger/src/history/hs_rec.c b/src/third_party/wiredtiger/src/history/hs_rec.c
index 6e30d425aec..15f856cc928 100644
--- a/src/third_party/wiredtiger/src/history/hs_rec.c
+++ b/src/third_party/wiredtiger/src/history/hs_rec.c
@@ -495,11 +495,10 @@ __wt_hs_insert_updates(
          * away.
          */
         modify_cnt = 0;
-        for (; updates.size > 0 &&
-             !(upd->txnid == list->onpage_upd->txnid &&
-               upd->start_ts == list->onpage_upd->start_ts);
-             tmp = full_value, full_value = prev_full_value, prev_full_value = tmp,
-             upd = prev_upd) {
+        for (; updates.size > 0; tmp = full_value, full_value = prev_full_value,
+                                 prev_full_value = tmp, upd = prev_upd) {
+            /* We should never insert the onpage value to the history store. */
+            WT_ASSERT(session, upd != list->onpage_upd);
             WT_ASSERT(session, upd->type == WT_UPDATE_STANDARD || upd->type == WT_UPDATE_MODIFY);
 
             tombstone = NULL;
@@ -651,11 +650,10 @@ __wt_hs_insert_updates(
             }
         }
 
-        /* If we squash the onpage value, there may be one or more updates left in the stack. */
-        if (updates.size > 0)
+        /* If we squash the onpage value, we increase the counter here. */
+        if (squashed)
             WT_STAT_CONN_DATA_INCR(session, cache_hs_write_squash);
 
-        __wt_update_vector_clear(&updates);
         /*
          * In the case that the onpage value is an out of order timestamp update and the update
          * older than it is a tombstone, it remains in the stack. Clean it up.
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index a2779a3fc63..5080fbc3952 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -1472,7 +1472,7 @@ extern int __wt_tiered_put_drop_shared(WT_SESSION_IMPL *session, WT_TIERED *tier
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_tiered_put_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[], bool reconfig)
+extern int __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[])
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_tiered_storage_destroy(WT_SESSION_IMPL *session)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
diff --git a/src/third_party/wiredtiger/src/tiered/tiered_config.c b/src/third_party/wiredtiger/src/tiered/tiered_config.c
index 6971ec4b7b5..8c0ec27333e 100644
--- a/src/third_party/wiredtiger/src/tiered/tiered_config.c
+++ b/src/third_party/wiredtiger/src/tiered/tiered_config.c
@@ -42,6 +42,8 @@ __tiered_common_config(WT_SESSION_IMPL *session, const char **cfg, WT_BUCKET_STO
 {
     WT_CONFIG_ITEM cval;
 
+    if (bstorage == NULL)
+        return (0);
     WT_RET(__wt_config_gets(session, cfg, "tiered_storage.local_retention", &cval));
     bstorage->retain_secs = (uint64_t)cval.val;
 
@@ -153,6 +155,8 @@ __wt_tiered_conn_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfi
 
     if (!reconfig)
         WT_RET(__wt_tiered_bucket_config(session, cfg, &conn->bstorage));
+    else
+        WT_ERR(__tiered_common_config(session, cfg, conn->bstorage));
 
     /* If the connection is not set up for tiered storage there is nothing more to do. */
     if (conn->bstorage == NULL)
@@ -161,13 +165,7 @@ __wt_tiered_conn_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfi
     __wt_verbose(
       session, WT_VERB_TIERED, "TIERED_CONFIG: prefix %s", conn->bstorage->bucket_prefix);
 
-    /*
-     * If reconfiguring, see if the other settings have changed on the system bucket storage.
-     */
     WT_ASSERT(session, conn->bstorage != NULL);
-    if (reconfig)
-        WT_ERR(__tiered_common_config(session, cfg, conn->bstorage));
-
     WT_STAT_CONN_SET(session, tiered_object_size, conn->bstorage->object_size);
     WT_STAT_CONN_SET(session, tiered_retention, conn->bstorage->retain_secs);
 
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index 1dbff451061..27233ca996c 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -2557,6 +2557,7 @@ buildvariants:
     - name: checkpoint-filetypes-test
     - name: unit-test-zstd
     - name: unit-test-long
+      distros: ubuntu1804-large
     - name: unit-test-random-seed
     - name: spinlock-gcc-test
     - name: spinlock-pthread-adaptive-test
@@ -2681,6 +2682,7 @@ buildvariants:
     - name: checkpoint-filetypes-test
     - name: unit-test-zstd
     - name: unit-test-long
+      distros: rhel80-large
     - name: spinlock-gcc-test
     - name: spinlock-pthread-adaptive-test
     - name: compile-wtperf
diff --git a/src/third_party/wiredtiger/test/suite/test_hs21.py b/src/third_party/wiredtiger/test/suite/test_hs21.py
index e2c8885661f..920dc41951b 100644
--- a/src/third_party/wiredtiger/test/suite/test_hs21.py
+++ b/src/third_party/wiredtiger/test/suite/test_hs21.py
@@ -47,7 +47,7 @@ class test_hs21(wttest.WiredTigerTestCase):
     session_config = 'isolation=snapshot'
     file_name = 'test_hs21'
     numfiles = 10
-    nrows = 10000
+    nrows = 1000
 
     def large_updates(self, uri, value, ds, nrows, commit_ts):
         # Update a large number of records, we'll hang if the history store table isn't working.
diff --git a/src/third_party/wiredtiger/test/suite/test_hs23.py b/src/third_party/wiredtiger/test/suite/test_hs23.py
new file mode 100644
index 00000000000..cfc7ad70623
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_hs23.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import wttest
+
+def timestamp_str(t):
+    return '%x' % t
+
+# test_hs23.py
+# Test the case that we have update, out of order timestamp
+# update, and update again in the same transaction
+class test_hs23(wttest.WiredTigerTestCase):
+    conn_config = 'cache_size=50MB'
+    session_config = 'isolation=snapshot'
+
+    def test(self):
+        uri = 'table:test_hs23'
+        self.session.create(uri, 'key_format=S,value_format=S')
+        cursor = self.session.open_cursor(uri)
+        self.conn.set_timestamp(
+            'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1))
+
+        value1 = 'a'
+        value2 = 'b'
+        value3 = 'c'
+        value4 = 'd'
+        value5 = 'e'
+
+        # Insert a key.
+        self.session.begin_transaction()
+        cursor[str(0)] = value1
+        self.session.commit_transaction('commit_timestamp=' + timestamp_str(2))
+
+        # Update at 10, update at 20, update at 15 (out of order), and
+        # update at 20 in the same transaction
+        self.session.begin_transaction()
+        cursor.set_key(str(0))
+        cursor.set_value(value2)
+        self.session.timestamp_transaction(
+            'commit_timestamp=' + timestamp_str(10))
+        self.assertEquals(cursor.update(), 0)
+
+        cursor.set_key(str(0))
+        cursor.set_value(value3)
+        self.session.timestamp_transaction(
+            'commit_timestamp=' + timestamp_str(20))
+        self.assertEquals(cursor.update(), 0)
+
+        cursor.set_key(str(0))
+        cursor.set_value(value4)
+        self.session.timestamp_transaction(
+            'commit_timestamp=' + timestamp_str(15))
+        self.assertEquals(cursor.update(), 0)
+
+        cursor.set_key(str(0))
+        cursor.set_value(value5)
+        self.session.timestamp_transaction(
+            'commit_timestamp=' + timestamp_str(20))
+        self.assertEquals(cursor.update(), 0)
+        self.session.commit_transaction()
+
+        # Do a checkpoint to trigger
+        # history store reconciliation.
+        self.session.checkpoint()
+
+        evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)")
+
+        # Search the key to evict it.
+        self.session.begin_transaction()
+        self.assertEqual(evict_cursor[str(0)], value5)
+        self.assertEqual(evict_cursor.reset(), 0)
+        self.session.rollback_transaction()
+
+        # Search the latest update
+        self.session.begin_transaction("read_timestamp=" + timestamp_str(20))
+        self.assertEqual(cursor[str(0)], value5)
+        self.session.rollback_transaction()
+
+        # Serarch the out of order timestamp update
+        self.session.begin_transaction("read_timestamp=" + timestamp_str(15))
+        self.assertEqual(cursor[str(0)], value4)
+        self.session.rollback_transaction()
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py
index e442fc23f14..26254f4d33d 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered04.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py
@@ -160,9 +160,11 @@ class test_tiered04(wttest.WiredTigerTestCase):
         self.conn.reconfigure(config)
         retain = self.get_stat(stat.conn.tiered_retention, None)
         self.assertEqual(retain, new)
-        #self.session.flush_tier(None)
-        #calls = self.get_stat(stat.conn.flush_tier, None)
-        #self.assertEqual(calls, 5)
+        self.pr("reconfigure flush_tier")
+        self.session.flush_tier(None)
+        self.pr("reconfigure get stat")
+        calls = self.get_stat(stat.conn.flush_tier, None)
+        self.assertEqual(calls, 5)
 
 if __name__ == '__main__':
     wttest.run()