From 2ea564e14717b2d091486e53b92c9ac58cad40f6 Mon Sep 17 00:00:00 2001 From: Chenhao Qu Date: Wed, 29 Dec 2021 13:08:34 +1100 Subject: Import wiredtiger: 79c2e12ea706ff1653154e87b2b51fe437be79a7 from branch mongodb-master ref: 936b411d68..79c2e12ea7 for: 5.3.0 WT-8561 Error if restoring a backup with metadata verification on --- src/third_party/wiredtiger/dist/api_data.py | 3 +- src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/conn/conn_api.c | 2 +- src/third_party/wiredtiger/src/include/extern.h | 2 +- .../wiredtiger/src/include/wiredtiger.in | 6 +- src/third_party/wiredtiger/src/meta/meta_turtle.c | 11 ++- src/third_party/wiredtiger/test/format/backup.c | 2 +- .../wiredtiger/test/suite/test_backup23.py | 101 +++++++++++++++++++++ 8 files changed, 120 insertions(+), 9 deletions(-) create mode 100755 src/third_party/wiredtiger/test/suite/test_backup23.py diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 8b3b11e05da..af6df028ffc 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1287,7 +1287,8 @@ wiredtiger_open_common =\ choices=['dsync', 'fsync', 'none']), ]), Config('verify_metadata', 'false', r''' - open connection and verify any WiredTiger metadata. This API + open connection and verify any WiredTiger metadata. Not compatible when + opening a connection from a backup. This API allows verification and detection of corruption in WiredTiger metadata.''', type='boolean'), Config('write_through', '', r''' diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index cddcd28d02b..4ccc0a78ca4 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "936b411d686b30f2b68d050914689ded27744ee5" + "commit": "79c2e12ea706ff1653154e87b2b51fe437be79a7" } diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index d96bf66d151..5bc1c269a3d 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2949,9 +2949,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c * THE TURTLE FILE MUST BE THE LAST FILE CREATED WHEN INITIALIZING THE DATABASE HOME, IT'S WHAT * WE USE TO DECIDE IF WE'RE CREATING OR NOT. */ - WT_ERR(__wt_turtle_init(session)); WT_ERR(__wt_config_gets(session, cfg, "verify_metadata", &cval)); verify_meta = cval.val; + WT_ERR(__wt_turtle_init(session, verify_meta)); /* Verify the metadata file. */ if (verify_meta) { diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 84630f39d4a..4cabe2ffacc 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1537,7 +1537,7 @@ extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_turtle_exists(WT_SESSION_IMPL *session, bool *existp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_turtle_init(WT_SESSION_IMPL *session) +extern int __wt_turtle_init(WT_SESSION_IMPL *session, bool verify_meta) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 6685f47da7d..45c0c7f0500 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -3138,9 +3138,9 @@ struct __wt_connection { * "read"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c "rts"\, \c "salvage"\, \c * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c "tiered"\, \c "timestamp"\, * \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; default \c [].} - * @config{verify_metadata, open connection and verify any WiredTiger metadata. This API allows - * verification and detection of corruption in WiredTiger metadata., a boolean flag; default \c - * false.} + * @config{verify_metadata, open connection and verify any WiredTiger metadata. Not compatible when + * opening a connection from a backup. This API allows verification and detection of corruption in + * WiredTiger metadata., a boolean flag; default \c false.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to files. Ignored on * non-Windows systems. Options are given as a list\, such as "write_through=[data]". * Configuring \c write_through requires care\, see @ref tuning_system_buffer_cache_direct_io for diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 8e95510f99a..324de0237db 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -223,7 +223,7 @@ __wt_turtle_exists(WT_SESSION_IMPL *session, bool *existp) * Check the turtle file and create if necessary. */ int -__wt_turtle_init(WT_SESSION_IMPL *session) +__wt_turtle_init(WT_SESSION_IMPL *session, bool verify_meta) { WT_DECL_RET; char *metaconf, *unused_value; @@ -307,6 +307,15 @@ __wt_turtle_init(WT_SESSION_IMPL *session) if (exist_incr) F_SET(S2C(session), WT_CONN_WAS_BACKUP); + /* + * Verifying the metadata is incompatible with restarting from a backup because the verify + * call will rewrite the metadata's checkpoint and could lead to skipping recovery. Test + * here before creating the metadata file and reading in the backup file. + */ + if (verify_meta && exist_backup) + WT_RET_MSG( + session, EINVAL, "restoring a backup is incompatible with metadata verification"); + /* Create the metadata file. */ WT_RET(__metadata_init(session)); diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index d865bd2f12f..3285037ef2f 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -65,7 +65,7 @@ check_copy(void) /* Now setup and open the path for real. */ testutil_check(__wt_snprintf(path, len, "%s/BACKUP", g.home)); - wts_open(path, &conn, &session, true); + wts_open(path, &conn, &session, false); /* Verify the objects. */ tables_apply(wts_verify, conn); diff --git a/src/third_party/wiredtiger/test/suite/test_backup23.py b/src/third_party/wiredtiger/test/suite/test_backup23.py new file mode 100755 index 00000000000..a54b0964f49 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_backup23.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import copy_wiredtiger_home +from suite_subprocess import suite_subprocess +import os +import shutil +import wiredtiger, wttest + +# test_backup.py +# Test error message if verifying metadata on a backup restore. +class test_backup23(wttest.WiredTigerTestCase, suite_subprocess): + '''Test backup, verifying metadata and an error opening the backup''' + + #conn_config = 'config_base=false,log=(archive=false,enabled),verbose=(recovery,log)' + conn_config = 'config_base=false,log=(archive=false,enabled)' + conn_config_err = 'config_base=false,log=(archive=false,enabled),verify_metadata=true' + #conn_config_err = 'config_base=false,log=(archive=false,enabled),verify_metadata=true,verbose=(recovery,log)' + dir='backup.dir' + nentries = 10 + uri = 'file:backup.wt' + + def take_full_backup(self, dir): + # Open up the backup cursor, and copy the files. Do a full backup. + cursor = self.session.open_cursor('backup:', None, None) + self.pr('Full backup to ' + dir + ': ') + os.mkdir(dir) + while True: + ret = cursor.next() + if ret != 0: + break + bkup_file = cursor.get_key() + sz = os.path.getsize(bkup_file) + self.pr('Copy from: ' + bkup_file + ' (' + str(sz) + ') to ' + dir) + shutil.copy(bkup_file, dir) + self.assertEqual(ret, wiredtiger.WT_NOTFOUND) + cursor.close() + + def test_backup23(self): + '''Test backup, verifying metadata and an error opening the backup''' + self.session.create(self.uri, 'key_format=i,value_format=i') + c = self.session.open_cursor(self.uri) + + self.session.begin_transaction() + for i in range(self.nentries): + c[i] = i + self.session.commit_transaction() + self.session.checkpoint() + + # Add more entries after the check point. They should be recovered. + self.session.begin_transaction() + for i in range(self.nentries): + c[i + self.nentries] = i + self.session.commit_transaction() + c.close() + orig_data = list(self.session.open_cursor(self.uri)) + + # Take a full backup. + self.take_full_backup(self.dir) + self.close_conn() + + msg = '/restoring a backup is incompatible/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open(self.dir, self.conn_config_err), msg) + + self.pr('try opening error directory with correct config') + # After getting the error we should be able to open the error backup directory with the + # correct setting and then also see our data. + self.conn = self.wiredtiger_open(self.dir, self.conn_config) + session = self.conn.open_session() + bkup_data = list(session.open_cursor(self.uri)) + + self.assertEqual(orig_data, bkup_data) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1