Import wiredtiger: fe32af9801bbfe192afeca271072250f35af6160 from branch mongodb-6.1

ref: c4fa0f7511..fe32af9801
for: 6.1.0-rc2

WT-9457 Preserve ckpt_most_recent value across restart (#8245)

9 files changed, 190 insertions, 76 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index e4e89543c08..34c007ed699 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-6.1",
-    "commit": "c4fa0f7511202f4b9908e0b70a444197dddd07c2"
+    "commit": "fe32af9801bbfe192afeca271072250f35af6160"
 }
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index 3da3943e9db..c40399357ab 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -3010,8 +3010,8 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c
         WT_ERR(wt_session->salvage(wt_session, WT_METAFILE_URI, NULL));
     }
 
-    /* Initialize the connection's base write generation. */
-    WT_ERR(__wt_metadata_init_base_write_gen(session));
+    /* Initialize connection values from stored metadata. */
+    WT_ERR(__wt_metadata_load_prior_state(session));
 
     WT_ERR(__wt_metadata_cursor(session, NULL));
     /*
diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c
index a396155d37c..a6487ab772f 100644
--- a/src/third_party/wiredtiger/src/conn/conn_open.c
+++ b/src/third_party/wiredtiger/src/conn/conn_open.c
@@ -39,8 +39,6 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[])
      */
     conn->default_session = session;
 
-    __wt_seconds(session, &conn->ckpt_most_recent);
-
     /*
      * Publish: there must be a barrier to ensure the connection structure fields are set before
      * other threads read from the pointer.
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index 01b4b9e2512..94a89044eef 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -1135,10 +1135,10 @@ extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cu
 extern int __wt_metadata_get_ckptlist(WT_SESSION *session, const char *name, WT_CKPT **ckptbasep)
   WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")))
     WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_metadata_init_base_write_gen(WT_SESSION_IMPL *session)
-  WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_metadata_insert(WT_SESSION_IMPL *session, const char *key, const char *value)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_metadata_load_prior_state(WT_SESSION_IMPL *session)
+  WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_metadata_remove(WT_SESSION_IMPL *session, const char *key)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep)
@@ -1147,7 +1147,7 @@ extern int __wt_metadata_turtle_rewrite(WT_SESSION_IMPL *session)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_metadata_update(WT_SESSION_IMPL *session, const char *key, const char *value)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_metadata_update_base_write_gen(WT_SESSION_IMPL *session, const char *config)
+extern int __wt_metadata_update_connection(WT_SESSION_IMPL *session, const char *config)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_modify_apply_api(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries)
   WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")))
diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c
index 60a783e86cd..3d105823e98 100644
--- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c
+++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c
@@ -12,9 +12,11 @@ static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *);
 static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **, int64_t *, uint64_t *);
 static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *);
 static int __ckpt_named(WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *);
+static int __ckpt_parse_time(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, uint64_t *);
 static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *, bool);
 static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *);
 static int __meta_blk_mods_load(WT_SESSION_IMPL *, const char *, WT_CKPT *, WT_CKPT *, bool);
+
 /*
  * __ckpt_load_blk_mods --
  *     Load the block information from the config string.
@@ -173,6 +175,30 @@ err:
 }
 
 /*
+ * __ckpt_parse_time --
+ *     Parse clock time from checkpoint metadata config. This requires special handling because
+ *     times are unsigned values and config parsing treats numeric values as signed.
+ */
+static int
+__ckpt_parse_time(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *config_value, uint64_t *timep)
+{
+    char timebuf[64];
+
+    WT_UNUSED(session);
+    *timep = 0;
+
+    if (config_value->len == 0 || config_value->len > sizeof(timebuf) - 1)
+        return (WT_ERROR);
+    memcpy(timebuf, config_value->str, config_value->len);
+    timebuf[config_value->len] = '\0';
+    /* NOLINTNEXTLINE(cert-err34-c) */
+    if (sscanf(timebuf, "%" SCNu64, timep) != 1)
+        return (WT_ERROR);
+
+    return (0);
+}
+
+/*
  * __wt_meta_checkpoint_by_name --
  *     Look up the requested named checkpoint in the metadata and return its order and time
  *     information.
@@ -213,7 +239,7 @@ __wt_meta_checkpoint_by_name(WT_SESSION_IMPL *session, const char *uri, const ch
             WT_ERR(__wt_config_subgets(session, &v, "write_gen", &a));
             if ((uint64_t)a.val >= conn->base_write_gen) {
                 WT_ERR(__wt_config_subgets(session, &v, "time", &a));
-                *timep = (uint64_t)a.val;
+                WT_ERR(__ckpt_parse_time(session, &a, timep));
             }
             break;
         }
@@ -372,12 +398,10 @@ __ckpt_last_name(WT_SESSION_IMPL *session, const char *config, const char **name
 {
     WT_CONFIG ckptconf;
     WT_CONFIG_ITEM a, k, v;
-    WT_CONNECTION_IMPL *conn;
     WT_DECL_RET;
     uint64_t time;
     int64_t found;
 
-    conn = S2C(session);
     *namep = NULL;
     time = 0;
 
@@ -391,13 +415,9 @@ __ckpt_last_name(WT_SESSION_IMPL *session, const char *config, const char **name
             continue;
         found = a.val;
 
-        /* If the write generation is current, extract the wall-clock time for matching purposes. */
-        WT_ERR(__wt_config_subgets(session, &v, "write_gen", &a));
-        if ((uint64_t)a.val >= conn->base_write_gen) {
-            WT_ERR(__wt_config_subgets(session, &v, "time", &a));
-            time = (uint64_t)a.val;
-        } else
-            time = 0;
+        /* Extract the wall-clock time for matching purposes. */
+        WT_ERR(__wt_config_subgets(session, &v, "time", &a));
+        WT_ERR(__ckpt_parse_time(session, &a, &time));
 
         __wt_free(session, *namep);
         WT_ERR(__wt_strndup(session, k.str, k.len, namep));
@@ -891,7 +911,6 @@ __ckpt_load(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v, WT_C
 {
     WT_CONFIG_ITEM a;
     WT_DECL_RET;
-    char timebuf[64];
 
     /*
      * Copy the name, address (raw and hex), order and time into the slot. If there's no address,
@@ -908,17 +927,13 @@ __ckpt_load(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v, WT_C
 
     WT_RET(__wt_config_subgets(session, v, "order", &a));
     if (a.len == 0)
-        goto format;
+        WT_RET_MSG(session, WT_ERROR, "corrupted order value in checkpoint config");
     ckpt->order = a.val;
 
     WT_RET(__wt_config_subgets(session, v, "time", &a));
-    if (a.len == 0 || a.len > sizeof(timebuf) - 1)
-        goto format;
-    memcpy(timebuf, a.str, a.len);
-    timebuf[a.len] = '\0';
-    /* NOLINTNEXTLINE(cert-err34-c) */
-    if (sscanf(timebuf, "%" SCNu64, &ckpt->sec) != 1)
-        goto format;
+    ret = __ckpt_parse_time(session, &a, &ckpt->sec);
+    if (ret != 0)
+        WT_RET_MSG(session, WT_ERROR, "corrupted time value in checkpoint config");
 
     WT_RET(__wt_config_subgets(session, v, "size", &a));
     ckpt->size = (uint64_t)a.val;
@@ -985,7 +1000,7 @@ __ckpt_load(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v, WT_C
 
     WT_RET(__wt_config_subgets(session, v, "write_gen", &a));
     if (a.len == 0)
-        goto format;
+        WT_RET_MSG(session, WT_ERROR, "corrupted write_gen in checkpoint config");
     ckpt->write_gen = (uint64_t)a.val;
 
     /*
@@ -999,17 +1014,15 @@ __ckpt_load(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v, WT_C
         ckpt->run_write_gen = (uint64_t)a.val;
 
     return (0);
-
-format:
-    WT_RET_MSG(session, WT_ERROR, "corrupted checkpoint list");
 }
 
 /*
- * __wt_metadata_update_base_write_gen --
- *     Update the connection's base write generation from the config string.
+ * __wt_metadata_update_connection --
+ *     Update the connection's base write generation and most recent checkpoint time from the config
+ *     string.
  */
 int
-__wt_metadata_update_base_write_gen(WT_SESSION_IMPL *session, const char *config)
+__wt_metadata_update_connection(WT_SESSION_IMPL *session, const char *config)
 {
     WT_CKPT ckpt;
     WT_CONNECTION_IMPL *conn;
@@ -1020,6 +1033,7 @@ __wt_metadata_update_base_write_gen(WT_SESSION_IMPL *session, const char *config
 
     if ((ret = __ckpt_last(session, config, &ckpt)) == 0) {
         conn->base_write_gen = WT_MAX(ckpt.write_gen + 1, conn->base_write_gen);
+        conn->ckpt_most_recent = WT_MAX(ckpt.sec, conn->ckpt_most_recent);
         __wt_meta_checkpoint_free(session, &ckpt);
     } else
         WT_RET_NOTFOUND_OK(ret);
@@ -1028,21 +1042,26 @@ __wt_metadata_update_base_write_gen(WT_SESSION_IMPL *session, const char *config
 }
 
 /*
- * __wt_metadata_init_base_write_gen --
- *     Initialize the connection's base write generation.
+ * __wt_metadata_load_prior_state --
+ *     Initialize the connection's base write generation and most recent checkpoint time.
  */
 int
-__wt_metadata_init_base_write_gen(WT_SESSION_IMPL *session)
+__wt_metadata_load_prior_state(WT_SESSION_IMPL *session)
 {
+    WT_CONNECTION_IMPL *conn;
     WT_DECL_RET;
     char *config;
 
+    conn = S2C(session);
+
     /* Initialize the base write gen to 1 */
-    S2C(session)->base_write_gen = 1;
+    conn->base_write_gen = 1;
+    /* Initialize most recent checkpoint time with current clock */
+    __wt_seconds(session, &conn->ckpt_most_recent);
     /* Retrieve the metadata entry for the metadata file. */
     WT_ERR(__wt_metadata_search(session, WT_METAFILE_URI, &config));
-    /* Update base write gen to the write gen of metadata. */
-    WT_ERR(__wt_metadata_update_base_write_gen(session, config));
+    /* Update base write gen and most recent checkpoint time from the metadata. */
+    WT_ERR(__wt_metadata_update_connection(session, config));
 
 err:
     __wt_free(session, config);
@@ -1071,8 +1090,8 @@ __wt_metadata_correct_base_write_gen(WT_SESSION_IMPL *session)
 
         WT_ERR(cursor->get_value(cursor, &config));
 
-        /* Update base write gen to the write gen. */
-        WT_ERR(__wt_metadata_update_base_write_gen(session, config));
+        /* Update base write gen and most recent checkpoint time. */
+        WT_ERR(__wt_metadata_update_connection(session, config));
     }
     WT_ERR_NOTFOUND_OK(ret, false);
 
@@ -1577,14 +1596,12 @@ __wt_meta_read_checkpoint_snapshot(WT_SESSION_IMPL *session, const char *ckpt_na
     WT_CONFIG list;
     WT_CONFIG_ITEM cval;
     WT_CONFIG_ITEM k;
-    WT_CONNECTION_IMPL *conn;
     WT_DECL_ITEM(tmp);
     WT_DECL_RET;
     uint64_t write_gen;
     uint32_t counter;
     char *sys_config;
 
-    conn = S2C(session);
     write_gen = 0;
     counter = 0;
     sys_config = NULL;
@@ -1660,14 +1677,11 @@ __wt_meta_read_checkpoint_snapshot(WT_SESSION_IMPL *session, const char *ckpt_na
         if (snap_write_gen != NULL)
             *snap_write_gen = write_gen;
 
-        /*
-         * If the write generation is current, extract the checkpoint time. Otherwise we use 0.
-         */
-        if (ckpttime != NULL && cval.val != 0 && write_gen >= conn->base_write_gen) {
+        /* Extract the checkpoint time. */
+        if (ckpttime != NULL) {
             WT_ERR_NOTFOUND_OK(
               __wt_config_getones(session, sys_config, WT_SYSTEM_CKPT_SNAPSHOT_TIME, &cval), false);
-            if (cval.val != 0)
-                *ckpttime = (uint64_t)cval.val;
+            WT_ERR(__ckpt_parse_time(session, &cval, ckpttime));
         }
 
         /*
@@ -1696,11 +1710,9 @@ __meta_retrieve_timestamp(WT_SESSION_IMPL *session, const char *system_uri,
   const char *timestamp_name, wt_timestamp_t *timestampp, uint64_t *ckpttime)
 {
     WT_CONFIG_ITEM cval;
-    WT_CONNECTION_IMPL *conn;
     WT_DECL_RET;
     char *sys_config;
 
-    conn = S2C(session);
     sys_config = NULL;
     *timestampp = WT_TXN_NONE;
     if (ckpttime != NULL)
@@ -1718,16 +1730,10 @@ __meta_retrieve_timestamp(WT_SESSION_IMPL *session, const char *system_uri,
         }
 
         if (ckpttime != NULL) {
-            /* If the write generation is current, extract the checkpoint time. Otherwise we use 0.
-             */
+            /* Extract the checkpoint time. */
             WT_ERR_NOTFOUND_OK(
-              __wt_config_getones(session, sys_config, WT_SYSTEM_TS_WRITE_GEN, &cval), false);
-            if (cval.val != 0 && (uint64_t)cval.val >= conn->base_write_gen) {
-                WT_ERR_NOTFOUND_OK(
-                  __wt_config_getones(session, sys_config, WT_SYSTEM_TS_TIME, &cval), false);
-                if (cval.val != 0)
-                    *ckpttime = (uint64_t)cval.val;
-            }
+              __wt_config_getones(session, sys_config, WT_SYSTEM_TS_TIME, &cval), false);
+            WT_ERR(__ckpt_parse_time(session, &cval, ckpttime));
         }
     }
 
diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c
index f9e3c52c850..c2d4c456bc6 100644
--- a/src/third_party/wiredtiger/src/session/session_dhandle.c
+++ b/src/third_party/wiredtiger/src/session/session_dhandle.c
@@ -462,15 +462,15 @@ __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri, const cha
      * actually zero in a newer, currently running checkpoint, because then they must have always
      * been zero.)
      *
-     * This scheme relies on the fact we take steps to make sure that the checkpoint wall clock time
-     * does not run backward, and that successive checkpoints are never given the same wall clock
-     * time. Note that we use the write generation to ignore wall clock times from previous database
-     * opens (all such are treated as 0) -- anything from a previous database open can't have been
-     * produced by a currently running checkpoint and can be presumed to match. This is done so we
-     * don't get in trouble if the system clock moves backwards between runs, and also to avoid
-     * possible issues if the checkpoint clock runs forward. (See notes about that in txn_ckpt.c.)
-     * Furthermore, this avoids any confusion potentially caused by older versions not including the
-     * checkpoint time in the snapshot and timestamp metadata.
+     * This scheme relies on the fact that the checkpoint wall clock time always moves forward. Each
+     * checkpoint is given a wall clock time at least one second greater than the previous
+     * checkpoint. Before recovery, we load the time of the last successful checkpoint in the
+     * previous database so we can ensure checkpoint times increase across restarts. This avoids
+     * trouble if the system clock moves backwards between runs, and also avoids possible issues if
+     * the checkpoint clock runs forward. (See comment about that in
+     * __txn_checkpoint_establish_time().) When reading from a previous database, the checkpoint
+     * time in the snapshot and timestamp metadata default to zero if not present, avoiding
+     * confusion caused by older versions that don't include these values.
      *
      * Also note that only the exact name "WiredTigerCheckpoint" needs to be resolved. Requests to
      * open specific versions, such as "WiredTigerCheckpoint.6", must be looked up like named
@@ -580,10 +580,7 @@ __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri, const cha
              * collection of checkpoint cursors they opened on different files all came from the
              * same global checkpoint or not. This is the same problem as checking if the history
              * store checkpoint and data store checkpoint match, so the wall time is the right thing
-             * to use for it. Note that it will be 0 for all checkpoints from before this run;
-             * however, it is impossible to open the same checkpoint name twice and get two
-             * different checkpoints from before the current database run, since the newer one must
-             * have just been created.
+             * to use for it.
              */
             ckpt_snapshot->ckpt_id = snapshot_time;
         }
diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c
index 775332a5568..827d5cab9ca 100644
--- a/src/third_party/wiredtiger/src/txn/txn_recover.c
+++ b/src/third_party/wiredtiger/src/txn/txn_recover.c
@@ -556,8 +556,8 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config)
       (WT_IS_MAX_LSN(&r->max_ckpt_lsn) || __wt_log_cmp(&lsn, &r->max_ckpt_lsn) > 0))
         WT_ASSIGN_LSN(&r->max_ckpt_lsn, &lsn);
 
-    /* Update the base write gen based on this file's configuration. */
-    if ((ret = __wt_metadata_update_base_write_gen(r->session, config)) != 0)
+    /* Update the base write gen and most recent checkpoint based on this file's configuration. */
+    if ((ret = __wt_metadata_update_connection(r->session, config)) != 0)
         WT_RET_MSG(r->session, ret, "Failed recovery setup for %s: cannot update write gen", uri);
     return (0);
 }
diff --git a/src/third_party/wiredtiger/test/suite/test_bug029.py b/src/third_party/wiredtiger/test/suite/test_bug029.py
new file mode 100644
index 00000000000..b2d9e1c6ac4
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_bug029.py
@@ -0,0 +1,112 @@
+
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# [TEST_TAGS]
+# checkpoint:recovery
+# [END_TAGS]
+
+import wttest
+import os, shutil
+
+# test_bug029.py
+#
+# Test that WT correctly propogates the most recent checkpoint time
+# across restarts. We validate this by reproducing the original bug
+# from WT-9457: frequent checkpoints pushed the checkpoint clock time
+# into the future such that immediately after a restart a backup could
+# see its checkpoint deleted out from under it. The result was fatal
+# read errors when restoring the backup.
+
+class test_bug029(wttest.WiredTigerTestCase):
+    conn_config = ("cache_size=50MB")
+    uri = "table:test_bug029"
+    bigvalue = "WiredTiger" * 100
+    backup_dir = "backup_dir"
+
+    def add_data(self, uri, start, count):
+        cursor = self.session.open_cursor(uri, None)
+        for i in range(start, start + count): 
+            cursor[i] = self.bigvalue
+        cursor.close()
+
+    def test_bug029(self):
+        # Create and populate table
+        self.session.create(self.uri, "key_format=i,value_format=S")
+        self.add_data(self.uri, 0, 2000)
+        self.session.checkpoint()
+
+        # Force the checkpoint time forward with a lot of quick checkpoints.
+        for i in range(100):
+            self.session.checkpoint("force=1")
+
+        # Add more data and checkpoint again. This creates a bunch of pages
+        # in the final checkpoint that can be deleted and reused if we hit
+        # the bug.
+        self.add_data(self.uri, 2000, 2000)
+        self.session.checkpoint()
+
+        # Shutdown and reopen.
+        self.reopen_conn()
+
+        self.add_data(self.uri, 0, 100)
+
+        # Open a backup cursor and force a few checkpoints. This will allow
+        # WT to delete older checkpoints, but as long as the backup cursor
+        # is open, it shouldn't delete the backup checkpoint---unless we hit
+        # the bug.
+        backup_cursor = self.session.open_cursor('backup:')
+
+        for i in range(10):
+            self.session.checkpoint("force=1")
+
+        # Write and checkpoint a bunch of data. If we erroneously deleted our 
+        # backup checkpoint this should overwrite some of that checkpoint's
+        # blocks.
+        self.add_data(self.uri, 1000, 2000)
+        self.session.checkpoint()
+
+        # Now do the backup.
+        os.mkdir(self.backup_dir)
+        while True:
+            ret = backup_cursor.next()
+            if ret != 0:
+                break
+            shutil.copy(backup_cursor.get_key(), self.backup_dir)
+        backup_cursor.close()
+
+        # Open the backup and read data. If the backup snapshot was corrupted
+        # we will panic and die here.
+        backup_conn = self.wiredtiger_open(self.backup_dir, self.conn_config)
+        session = backup_conn.open_session()
+        cur1 = session.open_cursor(self.uri)
+        for i in range(0, 4000, 10):
+            self.assertEqual(cur1[i], self.bigvalue)
+
+if __name__ == '__main__':
+    wttest.run()
diff --git a/src/third_party/wiredtiger/test/test_coverage.md b/src/third_party/wiredtiger/test/test_coverage.md
index cbaaafabd49..fe46317c57b 100644
--- a/src/third_party/wiredtiger/test/test_coverage.md
+++ b/src/third_party/wiredtiger/test/test_coverage.md
@@ -15,6 +15,7 @@
 |Checkpoint|History Store|[test_checkpoint03.py](../test/suite/test_checkpoint03.py)
 |Checkpoint|Metadata|[test_checkpoint_snapshot01.py](../test/suite/test_checkpoint_snapshot01.py)
 |Checkpoint|Obsolete Data|[test_checkpoint08.py](../test/suite/test_checkpoint08.py)
+|Checkpoint|Recovery|[test_bug029.py](../test/suite/test_bug029.py)
 |Compression||[test_dictionary.py](../test/suite/test_dictionary.py)
 |Config Api||[test_base02.py](../test/suite/test_base02.py), [test_config02.py](../test/suite/test_config02.py)
 |Connection Api||[test_version.py](../test/suite/test_version.py)