From 049759fb2a107b3e703dc74688e832e8b7b2f758 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Mon, 23 May 2022 05:12:19 +0000 Subject: Import wiredtiger: 973bff41fecccf49b1dc2542a408d239857edfa4 from branch mongodb-master ref: bf2d3db88d..973bff41fe for: 6.1.0-rc0 WT-9027 Implement support for WT_SESSION::alter for tiered storage --- src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/src/schema/schema_alter.c | 192 ++++++++++++++++++++- .../wiredtiger/src/schema/schema_worker.c | 7 - .../wiredtiger/test/suite/helper_tiered.py | 11 -- .../wiredtiger/test/suite/test_alter01.py | 4 +- .../wiredtiger/test/suite/test_alter02.py | 4 +- .../wiredtiger/test/suite/test_alter03.py | 21 +-- .../wiredtiger/test/suite/test_alter04.py | 5 +- .../wiredtiger/test/suite/test_alter05.py | 4 +- .../wiredtiger/test/suite/test_schema08.py | 4 +- 10 files changed, 213 insertions(+), 41 deletions(-) diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 868446bebaf..7653ce1911e 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "bf2d3db88d2b8513adcc55546f36bbd1de00aae1" + "commit": "973bff41fecccf49b1dc2542a408d239857edfa4" } diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index 3786a719c17..f44715ae25f 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -78,6 +78,26 @@ __alter_file(WT_SESSION_IMPL *session, const char *newcfg[]) return (__alter_apply(session, uri, newcfg, WT_CONFIG_BASE(session, file_meta))); } +/* + * __alter_tier -- + * Alter a tier. + */ +static int +__alter_tier(WT_SESSION_IMPL *session, const char *newcfg[]) +{ + const char *uri; + + /* + * We know that we have exclusive access to the tier. So it will be closed after we're done with + * it and the next open will see the updated metadata. + */ + uri = session->dhandle->name; + if (!WT_PREFIX_MATCH(uri, "tier:")) + return (__wt_unexpected_object_type(session, uri, "tier:")); + + return (__alter_apply(session, uri, newcfg, WT_CONFIG_BASE(session, tier_meta))); +} + /* * __alter_object -- * Alter a tiered object. There are no object dhandles. @@ -91,6 +111,173 @@ __alter_object(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) return (__alter_apply(session, uri, newcfg, WT_CONFIG_BASE(session, object_meta))); } +/* + * __alter_get_object_id_range -- + * Get current and oldest object IDs for the tiered object. + */ +static int +__alter_get_object_id_range(WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *uri, + uint32_t *current_idp, uint32_t *oldest_idp) +{ + WT_CONFIG_ITEM cval; + WT_DECL_RET; + uint32_t current_id, oldest_id; + char *value; + + *current_idp = *oldest_idp = WT_TIERED_OBJECTID_NONE; + value = NULL; + /* + * First try to get oldest and current object IDs from the tiered object. If it's not + * initialized, get this information from the metadata. + */ + current_id = tiered->current_id; + oldest_id = tiered->oldest_id; + if (current_id == WT_TIERED_OBJECTID_NONE) { + WT_RET(__wt_metadata_search(session, uri, &value)); + + WT_ERR(__wt_config_getones(session, value, "oldest", &cval)); + oldest_id = (uint32_t)cval.val; + + WT_ERR(__wt_config_getones(session, value, "last", &cval)); + current_id = (uint32_t)cval.val; + } + + *current_idp = current_id; + *oldest_idp = oldest_id; +err: + __wt_free(session, value); + + return ret; +} + +/* + * __alter_objects -- + * Alter all objects in the oldest-current range. + */ +static int +__alter_objects(WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *newcfg[], + uint32_t current_id, uint32_t oldest_id) +{ + WT_DECL_RET; + uint32_t object_id; + char *value; + const char *name; + + value = NULL; + name = NULL; + + /* + * Iterate over all objects in the range oldest-current object IDs and alter metadata of each of + * them. + */ + for (object_id = oldest_id; object_id < current_id; object_id++) { + WT_ERR(__wt_tiered_name(session, &tiered->iface, object_id, WT_TIERED_NAME_OBJECT, &name)); + + /* + * Check if this object present in the metadata. Skip it if not found. It is expected for + * the range of objects to be not contiguous, because some objects may have been deleted. + */ + ret = __wt_metadata_search(session, name, &value); + __wt_free(session, value); + WT_ERR_NOTFOUND_OK(ret, true); + if (ret == 0) { + WT_WITH_DHANDLE(session, NULL, ret = __schema_alter(session, name, newcfg)); + WT_ERR(ret); + } + __wt_free(session, name); + } +err: + __wt_free(session, name); + + return (ret); +} + +/* + * __alter_tiered -- + * Alter a tiered metadata. + */ +static int +__alter_tiered(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[], uint32_t flags) +{ + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; + WT_TIERED *tiered; + uint32_t current_id, name_flag, oldest_id; + u_int i; + char *value; + const char *name; + + dhandle = NULL; + value = NULL; + name = NULL; + + if (!WT_PREFIX_MATCH(uri, "tiered:")) + return (__wt_unexpected_object_type(session, uri, "tiered:")); + + /* + * If the operation requires exclusive access, close any open handles, including checkpoints. + */ + if (FLD_ISSET(flags, WT_DHANDLE_EXCLUSIVE)) { + WT_WITH_HANDLE_LIST_WRITE_LOCK( + session, ret = __wt_conn_dhandle_close_all(session, uri, false, false)); + WT_RET(ret); + } + + WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, flags)); + tiered = (WT_TIERED *)session->dhandle; + + WT_ERR(__alter_get_object_id_range(session, tiered, uri, ¤t_id, &oldest_id)); + + /* Alter each tier. */ + for (i = 0; i < WT_TIERED_MAX_TIERS; i++) { + dhandle = tiered->tiers[i].tier; + if (dhandle == NULL) { + /* + * Tiers may not be initialized because we open tiered handle with lock only flag. In + * this case we need to find the names of the tiers manually. + */ + if (i == WT_TIERED_INDEX_LOCAL) + name_flag = WT_TIERED_NAME_LOCAL; + else if (i == WT_TIERED_INDEX_SHARED) + name_flag = WT_TIERED_NAME_SHARED; + else + continue; + + WT_ERR(__wt_tiered_name(session, &tiered->iface, current_id, name_flag, &name)); + + /* Check if metadata has entry for this tier. */ + ret = __wt_metadata_search(session, name, &value); + __wt_free(session, value); + WT_ERR_NOTFOUND_OK(ret, true); + if (ret != 0) { + /* + * Not found in the metadata. Skip it. This is expected, for instance, in the + * scenario when tier hasn't been flushed and there's no shared tier yet. + */ + __wt_free(session, name); + continue; + } + } + + WT_WITH_DHANDLE(session, NULL, + ret = __schema_alter(session, name == NULL ? dhandle->name : name, newcfg)); + __wt_free(session, name); + WT_ERR(ret); + } + + /* Alter all objects. */ + WT_ERR(__alter_objects(session, tiered, newcfg, current_id, oldest_id)); + + /* Apply change to the tiered metadata. */ + WT_ERR(__alter_apply(session, uri, newcfg, WT_CONFIG_BASE(session, tiered_meta))); + +err: + WT_TRET(__wt_session_release_dhandle(session)); + __wt_free(session, name); + + return (ret); +} + /* * __alter_tree -- * Alter an index or colgroup reference. @@ -236,9 +423,10 @@ __schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) return (__alter_object(session, uri, newcfg)); if (WT_PREFIX_MATCH(uri, "table:")) return (__alter_table(session, uri, newcfg, exclusive_refreshed)); + if (WT_PREFIX_MATCH(uri, "tier:")) + return (__wt_exclusive_handle_operation(session, uri, __alter_tier, newcfg, flags)); if (WT_PREFIX_MATCH(uri, "tiered:")) - return (__wt_schema_tiered_worker(session, uri, __alter_file, NULL, newcfg, flags)); - + return (__alter_tiered(session, uri, newcfg, flags)); return (__wt_bad_object_type(session, uri)); } diff --git a/src/third_party/wiredtiger/src/schema/schema_worker.c b/src/third_party/wiredtiger/src/schema/schema_worker.c index 3ad0ef19a9b..6c3329ee0f1 100644 --- a/src/third_party/wiredtiger/src/schema/schema_worker.c +++ b/src/third_party/wiredtiger/src/schema/schema_worker.c @@ -49,13 +49,6 @@ __wt_schema_tiered_worker(WT_SESSION_IMPL *session, const char *uri, WT_TIERED *tiered; u_int i; - /* - * If this was an alter operation, we need to alter the configuration for the overall tree and - * then reread it so it isn't out of date. TODO not yet supported. - */ - if (FLD_ISSET(open_flags, WT_BTREE_ALTER)) - WT_RET(ENOTSUP); - WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, open_flags)); tiered = (WT_TIERED *)session->dhandle; diff --git a/src/third_party/wiredtiger/test/suite/helper_tiered.py b/src/third_party/wiredtiger/test/suite/helper_tiered.py index a7b59979921..983274da923 100644 --- a/src/third_party/wiredtiger/test/suite/helper_tiered.py +++ b/src/third_party/wiredtiger/test/suite/helper_tiered.py @@ -192,17 +192,6 @@ class TieredConfigMixin: extlist.skip_if_missing = True extlist.extension('storage_sources', self.ss_name + config) - # Wrapper around session.alter call - def alter(self, uri, alter_param): - # Tiered storage does not fully support alter operation. FIXME WT-9027. - try: - self.session.alter(uri, alter_param) - except BaseException as err: - if self.is_tiered_scenario() and str(err) == 'Operation not supported': - self.skipTest('Tiered storage does not fully support alter operation.') - else: - raise - def download_objects(self, bucket_name, prefix): import boto3 # The bucket from the storage source is expected to be a name and a region, separated by a diff --git a/src/third_party/wiredtiger/test/suite/test_alter01.py b/src/third_party/wiredtiger/test/suite/test_alter01.py index 30f6435fc8e..90f62397608 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter01.py +++ b/src/third_party/wiredtiger/test/suite/test_alter01.py @@ -164,7 +164,7 @@ class test_alter01(TieredConfigMixin, wttest.WiredTigerTestCase): cache_str = 'cache_resident=%s' % c alter_param += ',%s' % cache_str if alter_param != '': - self.alter(uri, alter_param) + self.session.alter(uri, alter_param) if self.reopen: self.reopen_conn() special = self.use_cg or self.use_index @@ -172,7 +172,7 @@ class test_alter01(TieredConfigMixin, wttest.WiredTigerTestCase): self.verify_metadata(access_str) self.verify_metadata(cache_str) else: - self.alter(suburi, alter_param) + self.session.alter(suburi, alter_param) self.verify_metadata(access_str) self.verify_metadata(cache_str) diff --git a/src/third_party/wiredtiger/test/suite/test_alter02.py b/src/third_party/wiredtiger/test/suite/test_alter02.py index 27cb4bd7bb9..13c0755ebca 100755 --- a/src/third_party/wiredtiger/test/suite/test_alter02.py +++ b/src/third_party/wiredtiger/test/suite/test_alter02.py @@ -220,9 +220,9 @@ class test_alter02(TieredConfigMixin, wttest.WiredTigerTestCase): self.conn.close() self.ConnectionOpen() - self.alter(uri, alter_param) + self.session.alter(uri, alter_param) if special: - self.alter(suburi, alter_param) + self.session.alter(suburi, alter_param) self.verify_metadata(log_str) # Put some more data in table. c = self.session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_alter03.py b/src/third_party/wiredtiger/test/suite/test_alter03.py index 8c7cff05d12..dcc5c6062a1 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter03.py +++ b/src/third_party/wiredtiger/test/suite/test_alter03.py @@ -66,7 +66,8 @@ class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase): # Removing quotes wrapping app metadata value just to make the test pass. # FIXME: WT-9036 - file_metastr = 'app_metadata=meta_data_1,' + if (file_metastr == 'app_metadata="meta_data_1",'): + file_metastr = 'app_metadata=meta_data_1,' else: c.set_key('file:' + self.name + '.wt') @@ -95,16 +96,16 @@ class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase): self.verify_metadata(app_meta_orig, '', app_meta_orig) # Alter app metadata and verify - self.alter(uri, 'app_metadata="meta_data_2",') + self.session.alter(uri, 'app_metadata="meta_data_2",') self.verify_metadata('app_metadata="meta_data_2",', '', 'app_metadata="meta_data_2",') # Alter app metadata, explicitly asking for exclusive access and verify - self.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,') + self.session.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,') self.verify_metadata('app_metadata="meta_data_3",', '', 'app_metadata="meta_data_3",') # Alter app metadata without taking exclusive lock and verify that only # table object gets modified - self.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,') + self.session.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,') self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') # Open a cursor, insert some data and try to alter with session open. @@ -114,15 +115,15 @@ class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase): c2[k+1] = 2 self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.alter(uri, 'app_metadata="meta_data_5",')) + lambda: self.session.alter(uri, 'app_metadata="meta_data_5",')) self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.alter(uri, + lambda: self.session.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_5",')) self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') - self.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,') + self.session.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,') self.verify_metadata('app_metadata="meta_data_5",', '', 'app_metadata="meta_data_3",') c2.close() @@ -145,17 +146,17 @@ class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase): # Try to alter app metadata without exclusive access and verify self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.alter(uri, + lambda: self.session.alter(uri, 'exclusive_refreshed=false,app_metadata="meta_data_2",'), '/is applicable only on simple tables/') self.verify_metadata('', 'app_metadata="meta_data_1",', '') # Alter app metadata, explicitly asking for exclusive access and verify - self.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",') + self.session.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",') self.verify_metadata('', 'app_metadata="meta_data_2",', '') # Alter app metadata and verify - self.alter(uri, 'app_metadata="meta_data_3",') + self.session.alter(uri, 'app_metadata="meta_data_3",') self.verify_metadata('', 'app_metadata="meta_data_3",', '') if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_alter04.py b/src/third_party/wiredtiger/test/suite/test_alter04.py index 8988deb754d..1512c293e8f 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter04.py +++ b/src/third_party/wiredtiger/test/suite/test_alter04.py @@ -134,14 +134,15 @@ class test_alter04(TieredConfigMixin, wttest.WiredTigerTestCase): # for all allowed settings. for a in self.cache_alter: alter_param = '%s=%s' % (self.setting, a) - self.alter(uri, alter_param) + self.session.alter(uri, alter_param) if self.reopen: self.reopen_conn() + special = self.use_cg or self.use_index if not special: self.verify_metadata(alter_param) else: - self.alter(suburi, alter_param) + self.session.alter(suburi, alter_param) self.verify_metadata(alter_param) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_alter05.py b/src/third_party/wiredtiger/test/suite/test_alter05.py index 7d2738d1ef9..47436361e62 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter05.py +++ b/src/third_party/wiredtiger/test/suite/test_alter05.py @@ -89,7 +89,7 @@ class test_alter05(TieredConfigMixin, wttest.WiredTigerTestCase): prev_alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint) # Alter the table and verify. - self.alter(uri, 'log=(enabled=false)') + self.session.alter(uri, 'log=(enabled=false)') self.verify_metadata('log=(enabled=false)') alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint) @@ -104,7 +104,7 @@ class test_alter05(TieredConfigMixin, wttest.WiredTigerTestCase): self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.alter(uri, 'log=(enabled=true)')) + lambda: self.session.alter(uri, 'log=(enabled=true)')) self.verify_metadata('log=(enabled=false)') alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint) diff --git a/src/third_party/wiredtiger/test/suite/test_schema08.py b/src/third_party/wiredtiger/test/suite/test_schema08.py index 1ff529a0a16..661bbd85d09 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema08.py +++ b/src/third_party/wiredtiger/test/suite/test_schema08.py @@ -72,9 +72,9 @@ class test_schema08(TieredConfigMixin, wttest.WiredTigerTestCase, suite_subproce def do_alter(self, uri, suburi): alter_param = 'cache_resident=true' - self.alter(uri, alter_param) + self.session.alter(uri, alter_param) if suburi != None: - self.alter(suburi, alter_param) + self.session.alter(suburi, alter_param) def do_ops(self, uri, suburi): if (self.schema_ops == 'none'): -- cgit v1.2.1