Import wiredtiger: 79c2e12ea706ff1653154e87b2b51fe437be79a7 from branch mongodb-master

ref: 936b411d68..79c2e12ea7
for: 5.3.0

WT-8561       Error if restoring a backup with metadata verification on

8 files changed, 120 insertions, 9 deletions

diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 8b3b11e05da..af6df028ffc 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -1287,7 +1287,8 @@ wiredtiger_open_common =\
             choices=['dsync', 'fsync', 'none']),
         ]),
     Config('verify_metadata', 'false', r'''
-        open connection and verify any WiredTiger metadata. This API
+        open connection and verify any WiredTiger metadata. Not compatible when
+        opening a connection from a backup. This API
         allows verification and detection of corruption in WiredTiger metadata.''',
         type='boolean'),
     Config('write_through', '', r'''
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index cddcd28d02b..4ccc0a78ca4 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "936b411d686b30f2b68d050914689ded27744ee5"
+    "commit": "79c2e12ea706ff1653154e87b2b51fe437be79a7"
 }
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index d96bf66d151..5bc1c269a3d 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -2949,9 +2949,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c
      * THE TURTLE FILE MUST BE THE LAST FILE CREATED WHEN INITIALIZING THE DATABASE HOME, IT'S WHAT
      * WE USE TO DECIDE IF WE'RE CREATING OR NOT.
      */
-    WT_ERR(__wt_turtle_init(session));
     WT_ERR(__wt_config_gets(session, cfg, "verify_metadata", &cval));
     verify_meta = cval.val;
+    WT_ERR(__wt_turtle_init(session, verify_meta));
 
     /* Verify the metadata file. */
     if (verify_meta) {
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index 84630f39d4a..4cabe2ffacc 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -1537,7 +1537,7 @@ extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_turtle_exists(WT_SESSION_IMPL *session, bool *existp)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
-extern int __wt_turtle_init(WT_SESSION_IMPL *session)
+extern int __wt_turtle_init(WT_SESSION_IMPL *session, bool verify_meta)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index 6685f47da7d..45c0c7f0500 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -3138,9 +3138,9 @@ struct __wt_connection {
  * "read"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "rts"\, \c "salvage"\, \c
  * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c "tiered"\, \c "timestamp"\,
  * \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; default \c [].}
- * @config{verify_metadata, open connection and verify any WiredTiger metadata.  This API allows
- * verification and detection of corruption in WiredTiger metadata., a boolean flag; default \c
- * false.}
+ * @config{verify_metadata, open connection and verify any WiredTiger metadata.  Not compatible when
+ * opening a connection from a backup.  This API allows verification and detection of corruption in
+ * WiredTiger metadata., a boolean flag; default \c false.}
  * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to files.  Ignored on
  * non-Windows systems.  Options are given as a list\, such as <code>"write_through=[data]"</code>.
  * Configuring \c write_through requires care\, see @ref tuning_system_buffer_cache_direct_io for
diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c
index 8e95510f99a..324de0237db 100644
--- a/src/third_party/wiredtiger/src/meta/meta_turtle.c
+++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c
@@ -223,7 +223,7 @@ __wt_turtle_exists(WT_SESSION_IMPL *session, bool *existp)
  *     Check the turtle file and create if necessary.
  */
 int
-__wt_turtle_init(WT_SESSION_IMPL *session)
+__wt_turtle_init(WT_SESSION_IMPL *session, bool verify_meta)
 {
     WT_DECL_RET;
     char *metaconf, *unused_value;
@@ -307,6 +307,15 @@ __wt_turtle_init(WT_SESSION_IMPL *session)
         if (exist_incr)
             F_SET(S2C(session), WT_CONN_WAS_BACKUP);
 
+        /*
+         * Verifying the metadata is incompatible with restarting from a backup because the verify
+         * call will rewrite the metadata's checkpoint and could lead to skipping recovery. Test
+         * here before creating the metadata file and reading in the backup file.
+         */
+        if (verify_meta && exist_backup)
+            WT_RET_MSG(
+              session, EINVAL, "restoring a backup is incompatible with metadata verification");
+
         /* Create the metadata file. */
         WT_RET(__metadata_init(session));
 
diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c
index d865bd2f12f..3285037ef2f 100644
--- a/src/third_party/wiredtiger/test/format/backup.c
+++ b/src/third_party/wiredtiger/test/format/backup.c
@@ -65,7 +65,7 @@ check_copy(void)
 
     /* Now setup and open the path for real. */
     testutil_check(__wt_snprintf(path, len, "%s/BACKUP", g.home));
-    wts_open(path, &conn, &session, true);
+    wts_open(path, &conn, &session, false);
 
     /* Verify the objects. */
     tables_apply(wts_verify, conn);
diff --git a/src/third_party/wiredtiger/test/suite/test_backup23.py b/src/third_party/wiredtiger/test/suite/test_backup23.py
new file mode 100755
index 00000000000..a54b0964f49
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_backup23.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+from helper import copy_wiredtiger_home
+from suite_subprocess import suite_subprocess
+import os
+import shutil
+import wiredtiger, wttest
+
+# test_backup.py
+#   Test error message if verifying metadata on a backup restore.
+class test_backup23(wttest.WiredTigerTestCase, suite_subprocess):
+    '''Test backup, verifying metadata and an error opening the backup'''
+
+    #conn_config = 'config_base=false,log=(archive=false,enabled),verbose=(recovery,log)'
+    conn_config = 'config_base=false,log=(archive=false,enabled)'
+    conn_config_err = 'config_base=false,log=(archive=false,enabled),verify_metadata=true'
+    #conn_config_err = 'config_base=false,log=(archive=false,enabled),verify_metadata=true,verbose=(recovery,log)'
+    dir='backup.dir'
+    nentries = 10
+    uri = 'file:backup.wt'
+
+    def take_full_backup(self, dir):
+        # Open up the backup cursor, and copy the files.  Do a full backup.
+        cursor = self.session.open_cursor('backup:', None, None)
+        self.pr('Full backup to ' + dir + ': ')
+        os.mkdir(dir)
+        while True:
+            ret = cursor.next()
+            if ret != 0:
+                break
+            bkup_file = cursor.get_key()
+            sz = os.path.getsize(bkup_file)
+            self.pr('Copy from: ' + bkup_file + ' (' + str(sz) + ') to ' + dir)
+            shutil.copy(bkup_file, dir)
+        self.assertEqual(ret, wiredtiger.WT_NOTFOUND)
+        cursor.close()
+
+    def test_backup23(self):
+        '''Test backup, verifying metadata and an error opening the backup'''
+        self.session.create(self.uri, 'key_format=i,value_format=i')
+        c = self.session.open_cursor(self.uri)
+
+        self.session.begin_transaction()
+        for i in range(self.nentries):
+            c[i] = i
+        self.session.commit_transaction()
+        self.session.checkpoint()
+
+        # Add more entries after the check point. They should be recovered.
+        self.session.begin_transaction()
+        for i in range(self.nentries):
+            c[i + self.nentries] = i
+        self.session.commit_transaction()
+        c.close()
+        orig_data = list(self.session.open_cursor(self.uri))
+
+        # Take a full backup.
+        self.take_full_backup(self.dir)
+        self.close_conn()
+
+        msg = '/restoring a backup is incompatible/'
+        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+            lambda: self.wiredtiger_open(self.dir, self.conn_config_err), msg)
+
+        self.pr('try opening error directory with correct config')
+        # After getting the error we should be able to open the error backup directory with the
+        # correct setting and then also see our data.
+        self.conn = self.wiredtiger_open(self.dir, self.conn_config)
+        session = self.conn.open_session()
+        bkup_data = list(session.open_cursor(self.uri))
+
+        self.assertEqual(orig_data, bkup_data)
+
+if __name__ == '__main__':
+    wttest.run()