Import wiredtiger: 57d44e1a1fb40776affd79d083b9c450ae989f64 from branch mongodb-4.4

ref: e64f50da56..57d44e1a1f
for: 4.4.11

WT-8534       Allow retrieving checkpoint snapshot for backup restore recovery
WT-8576       Enable logging in test checkpoint

5 files changed, 67 insertions, 16 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 02b3327da2b..2471a5a9ce6 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.4",
-    "commit": "e64f50da5638af780fbcda89a85d34c6847b1241"
+    "commit": "57d44e1a1fb40776affd79d083b9c450ae989f64"
 }
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index 7859eba5e8a..810546d69cb 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -2437,6 +2437,10 @@ __conn_version_verify(WT_SESSION_IMPL *session)
 
     conn = S2C(session);
 
+    conn->recovery_major = 0;
+    conn->recovery_minor = 0;
+    conn->recovery_patch = 0;
+
     /* Always set the compatibility versions. */
     __wt_logmgr_compat_version(session);
     /*
@@ -2445,7 +2449,12 @@ __conn_version_verify(WT_SESSION_IMPL *session)
     if (F_ISSET(conn, WT_CONN_SALVAGE))
         return (0);
 
-    /* If we have a turtle file, validate versions. */
+    /*
+     * Initialize the version variables. These aren't always populated since there are expected
+     * cases where the turtle files doesn't exist (restoring from a backup, for example). All code
+     * that deals with recovery versions must consider the case where they are default initialized
+     * to zero.
+     */
     WT_RET(__wt_fs_exist(session, WT_METADATA_TURTLE, &exist));
     if (exist)
         WT_RET(__wt_turtle_validate_version(session));
diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c
index f401332e588..d6d42f79d50 100644
--- a/src/third_party/wiredtiger/src/txn/txn_recover.c
+++ b/src/third_party/wiredtiger/src/txn/txn_recover.c
@@ -479,11 +479,21 @@ __recovery_set_checkpoint_snapshot(WT_SESSION_IMPL *session)
     conn = S2C(session);
     counter = 0;
 
+    /* Initialize the recovery checkpoint snapshot variables to default values. */
+    conn->recovery_ckpt_snap_min = WT_TXN_NONE;
+    conn->recovery_ckpt_snap_max = WT_TXN_NONE;
+    conn->recovery_ckpt_snapshot_count = 0;
+
     /*
-     * WiredTiger versions 10.0.1 onward have a valid checkpoint snapshot on-disk. Ignore reading
-     * the on-disk checkpoint snapshot from older versions.
+     * WiredTiger versions 10.0.1 onward have a valid checkpoint snapshot on-disk. There was a bug
+     * in some versions of WiredTiger that are tagged with the 10.0.0 release, which saved the wrong
+     * checkpoint snapshot (see WT-8395), so we ignore the snapshot when it was created with one of
+     * those versions. Versions of WiredTiger prior to 10.0.0 never saved a checkpoint snapshot.
+     * Additionally the turtle file doesn't always exist (for example, backup doesn't include the
+     * turtle file), so there isn't always a WiredTiger version available. If there is no version
+     * available, assume that the snapshot is valid, otherwise restoring from a backup won't work.
      */
-    if (conn->recovery_major < 10 ||
+    if ((conn->recovery_major != 0 && conn->recovery_major < 10) ||
       (conn->recovery_major == 10 && conn->recovery_minor == 0 && conn->recovery_patch == 0))
         return (0);
 
diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
index 924c7c33749..611a869cc6d 100644
--- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
+++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
@@ -293,8 +293,8 @@ wt_connect(const char *config_open)
           config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open));
     else {
         testutil_check(__wt_snprintf(config, sizeof(config),
-          "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),error_prefix="
-          "\"%s\"%s%s%s%s",
+          "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),log=(enabled),"
+          "error_prefix=\"%s\"%s%s%s%s",
           progname, g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",",
           config_open == NULL ? "" : config_open, timing_stress ? timing_stress_cofing : ""));
     }
diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py
index 9f796b28089..f9b299ea409 100644
--- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py
+++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py
@@ -29,7 +29,7 @@
 import fnmatch, os, shutil, threading, time
 from wtthread import checkpoint_thread, op_thread
 from helper import simulate_crash_restart
-import wttest
+import wiredtiger, wttest
 from wtdataset import SimpleDataSet
 from wtscenario import make_scenarios
 from wiredtiger import stat
@@ -43,18 +43,42 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase):
     # Create a table.
     uri = "table:test_checkpoint_snapshot02"
     nrows = 1000
+    backup_dir = "BACKUP"
+    backup_dir2 = "BACKUP2"
 
     key_format_values = [
         ('column', dict(key_format='r')),
         ('integer_row', dict(key_format='i')),
     ]
 
-    scenarios = make_scenarios(key_format_values)
+    restart_values = [
+        ("crash_restart", dict(restart=True)),
+        ("backup", dict(restart=False)),
+    ]
+
+    scenarios = make_scenarios(key_format_values, restart_values)
 
     def conn_config(self):
         config = 'cache_size=10MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[checkpoint_slow]'
         return config
 
+    def take_full_backup(self, fromdir, todir):
+        # Open up the backup cursor, and copy the files.  Do a full backup.
+        cursor = self.session.open_cursor('backup:', None, None)
+        self.pr('Full backup from '+ fromdir + ' to ' + todir + ': ')
+        os.mkdir(todir)
+        while True:
+            ret = cursor.next()
+            if ret != 0:
+                break
+            bkup_file = cursor.get_key()
+            copy_file = os.path.join(fromdir, bkup_file)
+            sz = os.path.getsize(copy_file)
+            self.pr('Copy from: ' + bkup_file + ' (' + str(sz) + ') to ' + todir)
+            shutil.copy(copy_file, todir)
+        self.assertEqual(ret, wiredtiger.WT_NOTFOUND)
+        cursor.close()
+
     def large_updates(self, uri, value, ds, nrows, commit_ts):
         # Update a large number of records.
         session = self.session
@@ -82,6 +106,15 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase):
         session.commit_transaction()
         self.assertEqual(count, nrows)
 
+    def perform_backup_or_crash_restart(self, fromdir, todir):
+        if self.restart == True:
+            #Simulate a crash by copying to a new directory(RESTART).
+            simulate_crash_restart(self, fromdir, todir)
+        else:
+            #Take a backup and restore it.
+            self.take_full_backup(fromdir, todir)
+            self.reopen_conn(todir)
+
     def test_checkpoint_snapshot(self):
 
         ds = SimpleDataSet(self, self.uri, 0, key_format=self.key_format, value_format="S",config='log=(enabled=false)')
@@ -113,8 +146,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase):
             done.set()
             ckpt.join()
 
-        #Simulate a crash by copying to a new directory(RESTART).
-        simulate_crash_restart(self, ".", "RESTART")
+        self.perform_backup_or_crash_restart(".", self.backup_dir)
 
         # Check the table contains the last checkpointed value.
         self.check(valuea, self.uri, self.nrows, 0)
@@ -166,8 +198,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase):
             done.set()
             ckpt.join()
 
-        #Simulate a crash by copying to a new directory(RESTART).
-        simulate_crash_restart(self, ".", "RESTART")
+        self.perform_backup_or_crash_restart(".", self.backup_dir)
 
         # Check the table contains the last checkpointed value.
         self.check(valuea, self.uri, self.nrows, 30)
@@ -224,8 +255,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase):
             ckpt.join()
 
         session1.rollback_transaction()
-        #Simulate a crash by copying to a new directory(RESTART).
-        simulate_crash_restart(self, ".", "RESTART")
+
+        self.perform_backup_or_crash_restart(".", self.backup_dir)
 
         # Check the table contains the last checkpointed value.
         self.check(valuea, self.uri, self.nrows, 30)
@@ -238,7 +269,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase):
         self.assertGreater(inconsistent_ckpt, 0)
         self.assertGreaterEqual(keys_removed, 0)
 
-        simulate_crash_restart(self, "RESTART", "RESTART2")
+        self.perform_backup_or_crash_restart(self.backup_dir, self.backup_dir2)
+
         # Check the table contains the last checkpointed value.
         self.check(valuea, self.uri, self.nrows, 30)