Import wiredtiger: fb7a60b6486c8c294c45cdc07fb1c3db86eb7424 from branch mongodb-master

ref: e3d8b3a085..fb7a60b648
for: 6.0.0

WT-8836       Tiered Tables can only copy files to object storage after a new checkpoint completes

19 files changed, 131 insertions, 65 deletions

diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 7623c57d8d8..1c17380a6b5 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -626,6 +626,11 @@ connection_runtime_config = [
             is not limited to not skewing newest, not favoring leaf pages,
             and modifying the eviction score mechanism.''',
             type='boolean'),
+        Config('flush_checkpoint', 'false', r'''
+            if true, call a system wide checkpoint immediately after a flush_tier 
+            completes to force objects out to disk so that a flush_tier can work
+            single-threaded''',
+            type='boolean'),
         Config('log_retention', '0', r'''
             adjust log removal to retain at least this number of log files, ignored if set to 0.
             (Warning: this option can remove log files required for recovery if no checkpoints
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index d9d704a0118..ce47de73b4b 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "e3d8b3a08567511fd48b2ca16f9a31be023b71c4"
+    "commit": "fb7a60b6486c8c294c45cdc07fb1c3db86eb7424"
 }
diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index a8297dd65b9..18b5df9b210 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -63,6 +63,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = {
   {"checkpoint_retention", "int", NULL, "min=0,max=1024", NULL, 0},
   {"corruption_abort", "boolean", NULL, NULL, NULL, 0},
   {"cursor_copy", "boolean", NULL, NULL, NULL, 0}, {"eviction", "boolean", NULL, NULL, NULL, 0},
+  {"flush_checkpoint", "boolean", NULL, NULL, NULL, 0},
   {"log_retention", "int", NULL, "min=0,max=1024", NULL, 0},
   {"realloc_exact", "boolean", NULL, NULL, NULL, 0},
   {"rollback_error", "int", NULL, "min=0,max=10M", NULL, 0},
@@ -128,7 +129,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = {
   {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2},
   {"compatibility", "category", NULL, NULL,
     confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1},
-  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10},
+  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 11},
   {"error_prefix", "string", NULL, NULL, NULL, 0},
   {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2},
   {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0},
@@ -856,7 +857,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = {
   {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0},
   {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3},
   {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0},
-  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10},
+  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 11},
   {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0},
   {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3},
   {"error_prefix", "string", NULL, NULL, NULL, 0},
@@ -938,7 +939,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = {
   {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0},
   {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3},
   {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0},
-  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10},
+  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 11},
   {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0},
   {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3},
   {"error_prefix", "string", NULL, NULL, NULL, 0},
@@ -1019,7 +1020,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = {
   {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2},
   {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0},
   {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3},
-  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10},
+  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 11},
   {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0},
   {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3},
   {"error_prefix", "string", NULL, NULL, NULL, 0},
@@ -1097,7 +1098,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = {
   {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2},
   {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0},
   {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3},
-  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 10},
+  {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 11},
   {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0},
   {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3},
   {"error_prefix", "string", NULL, NULL, NULL, 0},
@@ -1191,9 +1192,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     "cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB,"
     "checkpoint=(log_size=0,wait=0),compatibility=(release=),"
     "debug_mode=(checkpoint_retention=0,corruption_abort=true,"
-    "cursor_copy=false,eviction=false,log_retention=0,"
-    "realloc_exact=false,rollback_error=0,slow_checkpoint=false,"
-    "table_logging=false,update_restore_evict=false),error_prefix=,"
+    "cursor_copy=false,eviction=false,flush_checkpoint=false,"
+    "log_retention=0,realloc_exact=false,rollback_error=0,"
+    "slow_checkpoint=false,table_logging=false,"
+    "update_restore_evict=false),error_prefix=,"
     "eviction=(threads_max=8,threads_min=1),"
     "eviction_checkpoint_target=1,eviction_dirty_target=5,"
     "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
@@ -1470,8 +1472,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     "compatibility=(release=,require_max=,require_min=),"
     "config_base=true,create=false,debug_mode=(checkpoint_retention=0"
     ",corruption_abort=true,cursor_copy=false,eviction=false,"
-    "log_retention=0,realloc_exact=false,rollback_error=0,"
-    "slow_checkpoint=false,table_logging=false,"
+    "flush_checkpoint=false,log_retention=0,realloc_exact=false,"
+    "rollback_error=0,slow_checkpoint=false,table_logging=false,"
     "update_restore_evict=false),direct_io=,encryption=(keyid=,name=,"
     "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)"
     ",eviction_checkpoint_target=1,eviction_dirty_target=5,"
@@ -1510,8 +1512,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     "compatibility=(release=,require_max=,require_min=),"
     "config_base=true,create=false,debug_mode=(checkpoint_retention=0"
     ",corruption_abort=true,cursor_copy=false,eviction=false,"
-    "log_retention=0,realloc_exact=false,rollback_error=0,"
-    "slow_checkpoint=false,table_logging=false,"
+    "flush_checkpoint=false,log_retention=0,realloc_exact=false,"
+    "rollback_error=0,slow_checkpoint=false,table_logging=false,"
     "update_restore_evict=false),direct_io=,encryption=(keyid=,name=,"
     "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)"
     ",eviction_checkpoint_target=1,eviction_dirty_target=5,"
@@ -1549,12 +1551,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,"
     "compatibility=(release=,require_max=,require_min=),"
     "debug_mode=(checkpoint_retention=0,corruption_abort=true,"
-    "cursor_copy=false,eviction=false,log_retention=0,"
-    "realloc_exact=false,rollback_error=0,slow_checkpoint=false,"
-    "table_logging=false,update_restore_evict=false),direct_io=,"
-    "encryption=(keyid=,name=,secretkey=),error_prefix=,"
-    "eviction=(threads_max=8,threads_min=1),"
-    "eviction_checkpoint_target=1,eviction_dirty_target=5,"
+    "cursor_copy=false,eviction=false,flush_checkpoint=false,"
+    "log_retention=0,realloc_exact=false,rollback_error=0,"
+    "slow_checkpoint=false,table_logging=false,"
+    "update_restore_evict=false),direct_io=,encryption=(keyid=,name=,"
+    "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)"
+    ",eviction_checkpoint_target=1,eviction_dirty_target=5,"
     "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
     ",eviction_updates_target=0,eviction_updates_trigger=0,"
     "extensions=,file_extend=,file_manager=(close_handle_minimum=250,"
@@ -1587,12 +1589,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,"
     "compatibility=(release=,require_max=,require_min=),"
     "debug_mode=(checkpoint_retention=0,corruption_abort=true,"
-    "cursor_copy=false,eviction=false,log_retention=0,"
-    "realloc_exact=false,rollback_error=0,slow_checkpoint=false,"
-    "table_logging=false,update_restore_evict=false),direct_io=,"
-    "encryption=(keyid=,name=,secretkey=),error_prefix=,"
-    "eviction=(threads_max=8,threads_min=1),"
-    "eviction_checkpoint_target=1,eviction_dirty_target=5,"
+    "cursor_copy=false,eviction=false,flush_checkpoint=false,"
+    "log_retention=0,realloc_exact=false,rollback_error=0,"
+    "slow_checkpoint=false,table_logging=false,"
+    "update_restore_evict=false),direct_io=,encryption=(keyid=,name=,"
+    "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)"
+    ",eviction_checkpoint_target=1,eviction_dirty_target=5,"
     "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
     ",eviction_updates_target=0,eviction_updates_trigger=0,"
     "extensions=,file_extend=,file_manager=(close_handle_minimum=250,"
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index 8fd648cae8e..78ee338f34e 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -1157,7 +1157,6 @@ err:
      * the sweep server.
      */
     WT_TRET(__wt_sweep_destroy(session));
-    WT_TRET(__wt_tiered_storage_destroy(session));
 
     /*
      * Shut down the checkpoint and capacity server threads: we don't want to throttle writes and
@@ -1168,6 +1167,11 @@ err:
 
     /* Perform a final checkpoint and shut down the global transaction state. */
     WT_TRET(__wt_txn_global_shutdown(session, cfg));
+    /*
+     * Tiered storage needs to flush any work after the final checkpoint which happens when the
+     * global transaction state is shut down. So this shutdown must come after.
+     */
+    WT_TRET(__wt_tiered_storage_destroy(session));
 
     if (ret != 0) {
         __wt_err(session, ret, "failure during close, disabling further writes");
@@ -1975,6 +1979,12 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[])
     else
         F_CLR(cache, WT_CACHE_EVICT_DEBUG_MODE);
 
+    WT_RET(__wt_config_gets(session, cfg, "debug_mode.flush_checkpoint", &cval));
+    if (cval.val)
+        FLD_SET(conn->debug_flags, WT_CONN_DEBUG_FLUSH_CKPT);
+    else
+        FLD_CLR(conn->debug_flags, WT_CONN_DEBUG_FLUSH_CKPT);
+
     WT_RET(__wt_config_gets(session, cfg, "debug_mode.log_retention", &cval));
     conn->debug_log_cnt = (uint32_t)cval.val;
 
diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c
index 0d94e891d38..eb43e427b2c 100644
--- a/src/third_party/wiredtiger/src/conn/conn_tiered.c
+++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c
@@ -68,8 +68,10 @@ __flush_tier_wait(WT_SESSION_IMPL *session, const char **cfg)
         }
         if (++yield_count < WT_THOUSAND)
             __wt_yield();
-        else
+        else {
+            __wt_cond_signal(session, conn->tiered_cond);
             __wt_cond_wait(session, conn->flush_cond, 200, NULL);
+        }
     }
     return (0);
 }
@@ -111,7 +113,11 @@ __flush_tier_once(WT_SESSION_IMPL *session, uint32_t flags)
      */
     WT_ASSERT(session, FLD_ISSET(session->lock_flags, WT_SESSION_LOCKED_CHECKPOINT));
     __wt_seconds(session, &flush_time);
-    /* XXX If/when flush tier no longer requires the checkpoint lock, this needs consideration. */
+    /*
+     * XXX If/when flush tier no longer requires the checkpoint lock, all of these global values and
+     * their settings need consideration to make sure they don't race with a checkpoint.
+     */
+    conn->flush_ckpt_complete = false;
     conn->flush_most_recent = WT_MAX(flush_time, conn->ckpt_most_recent);
     conn->flush_ts = conn->txn_global.last_ckpt_timestamp;
 
@@ -436,6 +442,9 @@ __tier_storage_copy(WT_SESSION_IMPL *session)
     WT_DECL_RET;
     WT_TIERED_WORK_UNIT *entry;
 
+    /* There is nothing to do until the checkpoint after the flush completes. */
+    if (!S2C(session)->flush_ckpt_complete)
+        return (0);
     entry = NULL;
     for (;;) {
         /* Check if we're quitting or being reconfigured. */
@@ -496,6 +505,7 @@ __wt_flush_tier(WT_SESSION_IMPL *session, const char *config)
     WT_DECL_RET;
     uint32_t flags;
     const char *cfg[3];
+    const char *checkpoint_cfg[] = {WT_CONFIG_BASE(session, WT_SESSION_checkpoint), NULL};
     bool locked, wait;
 
     conn = S2C(session);
@@ -548,6 +558,8 @@ __wt_flush_tier(WT_SESSION_IMPL *session, const char *config)
           WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, ret = __flush_tier_once(session, flags)));
     __wt_spin_unlock(session, &conn->flush_tier_lock);
     locked = false;
+    if (FLD_ISSET(conn->debug_flags, WT_CONN_DEBUG_FLUSH_CKPT))
+        WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg, true));
 
     if (ret == 0 && LF_ISSET(WT_FLUSH_TIER_ON))
         WT_ERR(__flush_tier_wait(session, cfg));
@@ -806,6 +818,11 @@ __wt_tiered_storage_destroy(WT_SESSION_IMPL *session)
     /* Stop the internal server thread. */
     if (conn->flush_cond != NULL)
         __wt_cond_signal(session, conn->flush_cond);
+    if (conn->tiered_cond != NULL) {
+        __wt_cond_signal(session, conn->tiered_cond);
+        /* Give thread time to drain the work. */
+        __wt_sleep(1, 0);
+    }
     FLD_CLR(conn->server_flags, WT_CONN_SERVER_TIERED);
     if (conn->tiered_tid_set) {
         WT_ASSERT(session, conn->tiered_cond != NULL);
diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h
index 48aa7ccc50d..b681d54c471 100644
--- a/src/third_party/wiredtiger/src/include/connection.h
+++ b/src/third_party/wiredtiger/src/include/connection.h
@@ -430,7 +430,8 @@ struct __wt_connection_impl {
     WT_CONDVAR *flush_cond;          /* Flush wait mutex */
     WT_CONDVAR *tiered_cond;         /* Tiered wait mutex */
     bool tiered_server_running;      /* Internal tiered server operating */
-    uint64_t flush_most_recent;      /* Clock value of most recent flush_tier */
+    bool flush_ckpt_complete;        /* Checkpoint after flush completed */
+    uint64_t flush_most_recent;      /* Clock value of last flush_tier */
     uint32_t flush_state;            /* State of last flush tier */
     wt_timestamp_t flush_ts;         /* Timestamp of most recent flush_tier */
 
@@ -551,9 +552,10 @@ struct __wt_connection_impl {
 #define WT_CONN_DEBUG_CKPT_RETAIN 0x01u
 #define WT_CONN_DEBUG_CORRUPTION_ABORT 0x02u
 #define WT_CONN_DEBUG_CURSOR_COPY 0x04u
-#define WT_CONN_DEBUG_REALLOC_EXACT 0x08u
-#define WT_CONN_DEBUG_SLOW_CKPT 0x10u
-#define WT_CONN_DEBUG_UPDATE_RESTORE_EVICT 0x20u
+#define WT_CONN_DEBUG_FLUSH_CKPT 0x08u
+#define WT_CONN_DEBUG_REALLOC_EXACT 0x10u
+#define WT_CONN_DEBUG_SLOW_CKPT 0x20u
+#define WT_CONN_DEBUG_UPDATE_RESTORE_EVICT 0x40u
     /* AUTOMATIC FLAG VALUE GENERATION STOP 64 */
     uint64_t debug_flags;
 
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index b308e64cb41..9769df7661e 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -2145,30 +2145,34 @@ struct __wt_connection {
 	 * not limited to not skewing newest\, not favoring leaf pages\, and modifying the eviction
 	 * score mechanism., a boolean flag; default \c false.}
 	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;
-	 * log_retention, adjust log removal to retain at least this number of log files\, ignored
-	 * if set to 0. (Warning: this option can remove log files required for recovery if no
-	 * checkpoints have yet been done and the number of log files exceeds the configured value.
-	 * As WiredTiger cannot detect the difference between a system that has not yet checkpointed
-	 * and one that will never checkpoint\, it might discard log files before any checkpoint is
-	 * done.)., an integer between 0 and 1024; default \c 0.}
-	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;
-	 * realloc_exact, if true\, reallocation of memory will only provide the exact amount
-	 * requested.  This will help with spotting memory allocation issues more easily., a boolean
+	 * flush_checkpoint, if true\, call a system wide checkpoint immediately after a flush_tier
+	 * completes to force objects out to disk so that a flush_tier can work single-threaded., a
+	 * boolean flag; default \c false.}
+	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;log_retention, adjust
+	 * log removal to retain at least this number of log files\, ignored if set to 0. (Warning:
+	 * this option can remove log files required for recovery if no checkpoints have yet been
+	 * done and the number of log files exceeds the configured value.  As WiredTiger cannot
+	 * detect the difference between a system that has not yet checkpointed and one that will
+	 * never checkpoint\, it might discard log files before any checkpoint is done.)., an
+	 * integer between 0 and 1024; default \c 0.}
+	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;realloc_exact,
+	 * if true\, reallocation of memory will only provide the exact amount requested.  This will
+	 * help with spotting memory allocation issues more easily., a boolean flag; default \c
+	 * false.}
+	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;rollback_error, return a WT_ROLLBACK error from a
+	 * transaction operation about every Nth operation to simulate a collision., an integer
+	 * between 0 and 10M; default \c 0.}
+	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;slow_checkpoint, if
+	 * true\, slow down checkpoint creation by slowing down internal page processing., a boolean
+	 * flag; default \c false.}
+	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;table_logging, if true\, write
+	 * transaction related information to the log for all operations\, even operations for
+	 * tables with logging turned off.  This additional logging information is intended for
+	 * debugging and is informational only\, that is\, it is ignored during recovery., a boolean
+	 * flag; default \c false.}
+	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;update_restore_evict, if true\,
+	 * control all dirty page evictions through forcing update restore eviction., a boolean
 	 * flag; default \c false.}
-	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;rollback_error, return a
-	 * WT_ROLLBACK error from a transaction operation about every Nth operation to simulate a
-	 * collision., an integer between 0 and 10M; default \c 0.}
-	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;
-	 * slow_checkpoint, if true\, slow down checkpoint creation by slowing down internal page
-	 * processing., a boolean flag; default \c false.}
-	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;
-	 * table_logging, if true\, write transaction related information to the log for all
-	 * operations\, even operations for tables with logging turned off.  This additional logging
-	 * information is intended for debugging and is informational only\, that is\, it is ignored
-	 * during recovery., a boolean flag; default \c false.}
-	 * @config{&nbsp;&nbsp;&nbsp;&nbsp;
-	 * update_restore_evict, if true\, control all dirty page evictions through forcing update
-	 * restore eviction., a boolean flag; default \c false.}
 	 * @config{ ),,}
 	 * @config{error_prefix, prefix string for error messages., a string; default empty.}
 	 * @config{eviction = (, eviction configuration options., a set of related configuration
@@ -2863,15 +2867,19 @@ struct __wt_connection {
  * eviction, if true\, modify internal algorithms to change skew to force history store eviction to
  * happen more aggressively.  This includes but is not limited to not skewing newest\, not favoring
  * leaf pages\, and modifying the eviction score mechanism., a boolean flag; default \c false.}
- * @config{&nbsp;&nbsp;&nbsp;&nbsp;log_retention, adjust log removal to retain at least this number
- * of log files\, ignored if set to 0. (Warning: this option can remove log files required for
- * recovery if no checkpoints have yet been done and the number of log files exceeds the configured
- * value.  As WiredTiger cannot detect the difference between a system that has not yet checkpointed
- * and one that will never checkpoint\, it might discard log files before any checkpoint is done.).,
- * an integer between 0 and 1024; default \c 0.}
- * @config{&nbsp;&nbsp;&nbsp;&nbsp;realloc_exact, if
- * true\, reallocation of memory will only provide the exact amount requested.  This will help with
- * spotting memory allocation issues more easily., a boolean flag; default \c false.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;flush_checkpoint, if true\, call a system wide checkpoint
+ * immediately after a flush_tier completes to force objects out to disk so that a flush_tier can
+ * work single-threaded., a boolean flag; default \c false.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;
+ * log_retention, adjust log removal to retain at least this number of log files\, ignored if set to
+ * 0. (Warning: this option can remove log files required for recovery if no checkpoints have yet
+ * been done and the number of log files exceeds the configured value.  As WiredTiger cannot detect
+ * the difference between a system that has not yet checkpointed and one that will never
+ * checkpoint\, it might discard log files before any checkpoint is done.)., an integer between 0
+ * and 1024; default \c 0.}
+ * @config{&nbsp;&nbsp;&nbsp;&nbsp;realloc_exact, if true\, reallocation of
+ * memory will only provide the exact amount requested.  This will help with spotting memory
+ * allocation issues more easily., a boolean flag; default \c false.}
  * @config{&nbsp;&nbsp;&nbsp;&nbsp;rollback_error, return a WT_ROLLBACK error from a transaction
  * operation about every Nth operation to simulate a collision., an integer between 0 and 10M;
  * default \c 0.}
diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c
index 2855993e120..a763773bde8 100644
--- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c
+++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c
@@ -1152,10 +1152,12 @@ err:
 static int
 __txn_checkpoint_wrapper(WT_SESSION_IMPL *session, const char *cfg[])
 {
+    WT_CONNECTION_IMPL *conn;
     WT_DECL_RET;
     WT_TXN_GLOBAL *txn_global;
 
-    txn_global = &S2C(session)->txn_global;
+    conn = S2C(session);
+    txn_global = &conn->txn_global;
 
     WT_STAT_CONN_SET(session, txn_checkpoint_running, 1);
     txn_global->checkpoint_running = true;
@@ -1165,6 +1167,15 @@ __txn_checkpoint_wrapper(WT_SESSION_IMPL *session, const char *cfg[])
     WT_STAT_CONN_SET(session, txn_checkpoint_running, 0);
     txn_global->checkpoint_running = false;
 
+    /*
+     * Signal the tiered storage thread because it waits for the following checkpoint to complete to
+     * process flush units. Indicate that the checkpoint has completed.
+     */
+    if (conn->tiered_cond != NULL) {
+        conn->flush_ckpt_complete = true;
+        __wt_cond_signal(session, conn->tiered_cond);
+    }
+
     return (ret);
 }
 
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py
index 725fefca85c..9887de11055 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered02.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py
@@ -53,6 +53,7 @@ class test_tiered02(wttest.WiredTigerTestCase):
         if self.ss_name == 'local_store' and not os.path.exists(self.bucket):
             os.mkdir(self.bucket)
         return \
+          'debug_mode=(flush_checkpoint=true),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
           'bucket_prefix=%s,' % self.bucket_prefix + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py
index 47e614d340b..3b7dacabf71 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered03.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py
@@ -82,6 +82,7 @@ class test_tiered03(wttest.WiredTigerTestCase):
                 os.mkdir(self.absolute_bucket_dir)
             bucket_ret = self.absolute_bucket_dir
         return \
+          'debug_mode=(flush_checkpoint=true),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % bucket_ret  + \
           'cache_directory=%s,' % cache_dir + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py
index aebd20299aa..b5a00a26b21 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered04.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py
@@ -76,6 +76,7 @@ class test_tiered04(wttest.WiredTigerTestCase):
             os.mkdir(self.bucket)
             os.mkdir(self.bucket1)
         self.saved_conn = \
+          'debug_mode=(flush_checkpoint=true),' + \
           'statistics=(all),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered07.py b/src/third_party/wiredtiger/test/suite/test_tiered07.py
index 275b9d04f01..b22b8fd158a 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered07.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered07.py
@@ -74,6 +74,7 @@ class test_tiered07(wttest.WiredTigerTestCase):
         #  'verbose=(tiered),' + \
 
         return \
+          'debug_mode=(flush_checkpoint=true),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
           'bucket_prefix=%s,' % self.bucket_prefix + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered08.py b/src/third_party/wiredtiger/test/suite/test_tiered08.py
index 903fb79a688..384b9a4d527 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered08.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered08.py
@@ -65,6 +65,7 @@ class test_tiered08(wttest.WiredTigerTestCase):
         if self.ss_name == 'local_store' and not os.path.exists(self.bucket):
             os.mkdir(self.bucket)
         return \
+          'debug_mode=(flush_checkpoint=true),' + \
           'statistics=(fast),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered09.py b/src/third_party/wiredtiger/test/suite/test_tiered09.py
index 984da29ce91..8b411623234 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered09.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered09.py
@@ -66,6 +66,7 @@ class test_tiered09(wttest.WiredTigerTestCase):
         if self.ss_name == 'local_store' and not os.path.exists(self.bucket):
             os.mkdir(self.bucket)
         self.saved_conn = \
+          'debug_mode=(flush_checkpoint=true),' + \
           'statistics=(all),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered10.py b/src/third_party/wiredtiger/test/suite/test_tiered10.py
index d82f7e56c8f..06ca883c19e 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered10.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered10.py
@@ -76,6 +76,7 @@ class test_tiered10(wttest.WiredTigerTestCase):
         bucket += self.bucket
 
         self.saved_conn = \
+          'debug_mode=(flush_checkpoint=true),' + \
           'create,statistics=(all),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % bucket + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered11.py b/src/third_party/wiredtiger/test/suite/test_tiered11.py
index 1e3342ca79f..5afd06301f6 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered11.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered11.py
@@ -58,6 +58,7 @@ class test_tiered11(wttest.WiredTigerTestCase):
         if self.ss_name == 'local_store' and not os.path.exists(self.bucket):
             os.mkdir(self.bucket)
         self.saved_conn = \
+          'debug_mode=(flush_checkpoint=true),' + \
           'statistics=(all),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered12.py b/src/third_party/wiredtiger/test/suite/test_tiered12.py
index ecc040a6e51..5f53f79e00b 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered12.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered12.py
@@ -58,6 +58,7 @@ class test_tiered12(wttest.WiredTigerTestCase):
         if self.ss_name == 'local_store' and not os.path.exists(self.bucket):
             os.mkdir(self.bucket)
         self.saved_conn = \
+          'debug_mode=(flush_checkpoint=true),' + \
           'statistics=(all),timing_stress_for_test=(tiered_flush_finish),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered13.py b/src/third_party/wiredtiger/test/suite/test_tiered13.py
index e62649638cd..cc93d3dc9bc 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered13.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered13.py
@@ -76,6 +76,7 @@ class test_tiered13(test_import_base):
         if self.ss_name == 'local_store' and not os.path.exists(self.bucket):
             os.mkdir(self.bucket)
         self.saved_conn = \
+          'debug_mode=(flush_checkpoint=true),' + \
           'create,tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
           'bucket_prefix=%s,' % self.bucket_prefix + \
diff --git a/src/third_party/wiredtiger/test/suite/test_tiered14.py b/src/third_party/wiredtiger/test/suite/test_tiered14.py
index fe3f7b7787f..5d36b8667e2 100755
--- a/src/third_party/wiredtiger/test/suite/test_tiered14.py
+++ b/src/third_party/wiredtiger/test/suite/test_tiered14.py
@@ -75,6 +75,7 @@ class test_tiered14(wttest.WiredTigerTestCase):
         if self.ss_name == 'local_store' and not os.path.exists(self.bucket):
             os.mkdir(self.bucket)
         return \
+          'debug_mode=(flush_checkpoint=true),' + \
           'tiered_storage=(auth_token=%s,' % self.auth_token + \
           'bucket=%s,' % self.bucket + \
           'bucket_prefix=%s,' % self.bucket_prefix + \