diff options
7 files changed, 180 insertions, 9 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 7eacd529961..8c0d04fcf7e 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "5c62957734fe457d109cf95fa3f0567c22948a4b" + "commit": "2a6c384e7869c310e970ec3ce0fcce2faf75e24e" } diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 97207244a82..4030c19708b 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -28,6 +28,13 @@ __wt_bm_read( #ifdef HAVE_DIAGNOSTIC /* + * In diagnostic mode, verify the user isn't trying to import a tiered object as a regular table + * file. + */ + if (objectid != 0 && !WT_PREFIX_MATCH(block->name, "tiered:") && + !WT_SUFFIX_MATCH(block->name, ".wtobj")) + WT_RET_MSG(session, ENOTSUP, "%s: is a tiered object", block->name); + /* * In diagnostic mode, verify the block we're about to read isn't on the available list, or for * live systems, the discard list. */ diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index bd1ec73b2fc..38cf8588757 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -189,6 +189,11 @@ __create_file( * reconstruct the configuration metadata from the file. */ if (import) { + /* + * FIXME-WT-7735: Importing a tiered table is not yet allowed. + */ + if (WT_SUFFIX_MATCH(filename, ".wtobj")) + WT_ERR_MSG(session, ENOTSUP, "%s: import not supported on tiered files", uri); /* First verify that the data to import exists on disk. */ WT_IGNORE_RET(__wt_fs_exist(session, filename, &exists)); if (!exists) @@ -209,6 +214,12 @@ __create_file( cval.len -= 2; } WT_ERR(__wt_strndup(session, cval.str, cval.len, &filemeta)); + /* + * FIXME-WT-7735: Importing a tiered table is not yet allowed. + */ + if (__wt_config_getones(session, filemeta, "tiered_object", &cval) == 0 && + cval.val != 0) + WT_ERR_MSG(session, ENOTSUP, "%s: import not supported on tiered files", uri); filecfg[2] = filemeta; /* * If there is a file metadata provided, reconstruct the incremental backup diff --git a/src/third_party/wiredtiger/test/suite/test_tiered09.py b/src/third_party/wiredtiger/test/suite/test_tiered09.py index 507eb31cd85..21be523dc7b 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered09.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered09.py @@ -37,13 +37,9 @@ class test_tiered09(wttest.WiredTigerTestCase): # If the 'uri' changes all the other names must change with it. base = 'test_tiered09-000000000' base2 = 'test_second09-000000000' - fileuri_base = 'file:' + base obj1file = base + '1.wtobj' obj1second = base2 + '1.wtobj' obj2file = base + '2.wtobj' - obj3file = base + '3.wtobj' - objuri = 'object:' + base + '1.wtobj' - tiereduri = "tiered:test_tiered09" uri = "table:test_tiered09" uri2 = "table:test_second09" diff --git a/src/third_party/wiredtiger/test/suite/test_tiered10.py b/src/third_party/wiredtiger/test/suite/test_tiered10.py index 5b851000d12..2e634ec6c18 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered10.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered10.py @@ -37,10 +37,7 @@ class test_tiered10(wttest.WiredTigerTestCase): # If the 'uri' changes all the other names must change with it. base = 'test_tiered10-000000000' - fileuri_base = 'file:' + base obj1file = base + '1.wtobj' - objuri = 'object:' + base + '1.wtobj' - tiereduri = "tiered:test_tiered10" uri = "table:test_tiered10" auth_token = "test_token" diff --git a/src/third_party/wiredtiger/test/suite/test_tiered11.py b/src/third_party/wiredtiger/test/suite/test_tiered11.py index 341ab02cb14..adfe45926dc 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered11.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered11.py @@ -35,7 +35,6 @@ StorageSource = wiredtiger.StorageSource # easy access to constants class test_tiered11(wttest.WiredTigerTestCase): # If the 'uri' changes all the other names must change with it. base = 'test_tiered11-000000000' - fileuri_base = 'file:' + base nentries = 10 objuri = 'object:' + base + '1.wtobj' tiereduri = "tiered:test_tiered11" diff --git a/src/third_party/wiredtiger/test/suite/test_tiered13.py b/src/third_party/wiredtiger/test/suite/test_tiered13.py new file mode 100755 index 00000000000..08f83d2bf25 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_tiered13.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_tiered13.py +# Check that importing tiered tables returns an error. + +import os, shutil, wiredtiger +from test_import01 import test_import_base + +class test_tiered13(test_import_base): + # If the 'uri' changes all the other names must change with it. + base = 'test_tiered13-000000000' + fileuri_base = 'file:' + base + file1uri = fileuri_base + '1.wtobj' + file2 = base + '2.wtobj' + file2uri = fileuri_base + '2.wtobj' + otherfile = 'other.wt' + otheruri = 'file:' + otherfile + uri = "table:test_tiered13" + + auth_token = "test_token" + bucket = "my_bucket" + bucket_prefix = "my_prefix" + extension_name = "local_store" + + def conn_extensions(self, extlist): + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True + extlist.extension('storage_sources', self.extension_name) + + def conn_config(self): + os.mkdir(self.bucket) + self.saved_conn = \ + 'create,tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.bucket_prefix + \ + 'name=%s,' % self.extension_name + \ + 'object_target_size=20M),' + return self.saved_conn + + def test_tiered13(self): + # Create a new tiered table. + self.session.create(self.uri, 'key_format=S,value_format=S,') + # Add first data. Checkpoint, flush and close the connection. + c = self.session.open_cursor(self.uri) + c["0"] = "0" + c.close() + self.session.checkpoint() + self.session.flush_tier(None) + c = self.session.open_cursor(self.uri) + c["1"] = "1" + c.close() + self.session.checkpoint() + # We now have the second object existing, with data in it. + + # Set up for the test. + # - Create the tiered table (above). + # - Find the metadata for the current file: object. + # - Set up a new database for importing. + # + # Testing import and tiered tables. All should error: + # - Try to import via the table:uri. + # - Try to import via the table:uri with the file object's metadata. + # - Try to import via the file:uri. + # - Try to import via the file:uri with the file object's metadata. + # - Try to import via a renamed file:name.wt. + # - Try to import via a renamed file:name.wt with the file object's metadata. + + # Export the metadata for the current file object 2. + cursor = self.session.open_cursor('metadata:', None, None) + for k, v in cursor: + if k.startswith(self.file2uri): + fileobj_config = cursor[k] + if k.startswith(self.uri): + table_config = cursor[k] + cursor.close() + self.close_conn() + # Contruct the config strings. + import_enabled = 'import=(enabled,repair=true)' + import_meta = 'import=(enabled,repair=false,file_metadata=(' + \ + fileobj_config + '))' + table_import_meta = table_config + ',import=(enabled,repair=false,file_metadata=(' + \ + fileobj_config + '))' + + # Set up the import database. + newdir = 'IMPORT_DB' + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + newbucket = os.path.join(newdir, self.bucket) + os.mkdir(newbucket) + # It is tricky to work around the extension and connection bucket setup for + # creating the new import directory that is tiered-enabled. + ext = self.extensionsConfig() + conn_params = self.saved_conn + ext + self.conn = self.wiredtiger_open(newdir, conn_params) + self.session = self.setUpSessionOpen(self.conn) + + # Copy the file to the file names we're going to test later. + self.copy_file(self.file2, '.', newdir) + copy_from = self.file2 + copy_to = os.path.join(newdir, self.otherfile) + shutil.copy(copy_from, copy_to) + + msg = '/Operation not supported/' + enoent = '/No such file/' + # Try to import via the table:uri. This fails with ENOENT because + # it is looking for the normal on-disk file name. It cannot tell it + # is a tiered table in this case. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.create(self.uri, import_enabled), enoent) + # Try to import via the table:uri with file metadata. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.create(self.uri, table_import_meta), msg) + # Try to import via the file:uri. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.create(self.file2uri, import_enabled), msg) + # Try to import via the file:uri with file metadata. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.create(self.file2uri, import_meta), msg) + + # Try to import via a renamed object. If we don't send in metadata, + # we cannot tell it was a tiered table until we read in the root page. + # Only test this in diagnostic mode which has an assertion. + # + # FIXME-8644 There is an error path bug in wt_bm_read preventing this from + # working correctly although the code to return an error is in the code. + # Uncomment these lines when that bug is fixed. + + #if wiredtiger.diagnostic_build(): + # self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + # lambda: self.session.create(self.otheruri, import_enabled), msg) + + # Try to import via a renamed object with metadata. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.create(self.otheruri, import_meta), msg) |