From b0ffe36c691a83dee1587c248148231b5d9f62c1 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Mon, 4 Apr 2022 13:29:24 +1000 Subject: Import wiredtiger: 930ccee6ffc8f9f1f2ff77e74d1b47be150c2409 from branch mongodb-master ref: c576dfa728..930ccee6ff for: 6.0.0-rc0 WT-9026 Add tiered storage scenario to all test_alter python tests --- src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/test/suite/helper_tiered.py | 77 ++++++++++++++++++++++ .../wiredtiger/test/suite/test_alter01.py | 20 ++++-- .../wiredtiger/test/suite/test_alter02.py | 21 ++++-- .../wiredtiger/test/suite/test_alter03.py | 38 +++++++---- .../wiredtiger/test/suite/test_alter04.py | 12 ++-- .../wiredtiger/test/suite/test_alter05.py | 26 +++++--- 7 files changed, 162 insertions(+), 34 deletions(-) diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index f60840276a0..ea87ab9a76b 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "c576dfa7288511efa2ea82865491ad1f429d3516" + "commit": "930ccee6ffc8f9f1f2ff77e74d1b47be150c2409" } diff --git a/src/third_party/wiredtiger/test/suite/helper_tiered.py b/src/third_party/wiredtiger/test/suite/helper_tiered.py index 64406130d63..bb252f588ea 100644 --- a/src/third_party/wiredtiger/test/suite/helper_tiered.py +++ b/src/third_party/wiredtiger/test/suite/helper_tiered.py @@ -90,3 +90,80 @@ def generate_s3_prefix(test_name = ''): return prefix +# Storage sources. +tiered_storage_sources = [ + ('dirstore', dict(is_tiered = True, + is_local_storage = True, + auth_token = get_auth_token('dir_store'), + bucket = get_bucket1_name('dir_store'), + bucket_prefix = "pfx_", + ss_name = 'dir_store')), + ('s3', dict(is_tiered = True, + is_local_storage = False, + auth_token = get_auth_token('s3_store'), + bucket = get_bucket1_name('s3_store'), + bucket_prefix = generate_s3_prefix(), + ss_name = 's3_store')), + ('non_tiered', dict(is_tiered = False)), +] + +# This mixin class provides tiered storage configuration methods. +class TieredConfigMixin: + # Returns True if the current scenario is tiered. + def is_tiered_scenario(self): + return hasattr(self, 'is_tiered') and self.is_tiered + + # Setup connection config. + def conn_config(self): + return self.tiered_conn_config() + + # Setup tiered connection config. + def tiered_conn_config(self): + # Handle non_tiered storage scenarios. + if not self.is_tiered_scenario(): + return '' + + # Setup directories structure for local tiered storage. + if self.is_local_storage and not os.path.exists(self.bucket): + os.mkdir(self.bucket) + + # Build tiered storage connection string. + return \ + 'debug_mode=(flush_checkpoint=true),' + \ + 'tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.bucket_prefix + \ + 'name=%s),tiered_manager=(wait=0)' % self.ss_name + + # Load the storage sources extension. + def conn_extensions(self, extlist): + return self.tiered_conn_extensions(extlist) + + # Load tiered storage source extension. + def tiered_conn_extensions(self, extlist): + # Handle non_tiered storage scenarios. + if not self.is_tiered_scenario(): + return '' + + config = '' + # S3 store is built as an optional loadable extension, not all test environments build S3. + if not self.is_local_storage: + #config = '=(config=\"(verbose=1)\")' + extlist.skip_if_missing = True + #if self.is_local_storage: + #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' + # Windows doesn't support dynamically loaded extension libraries. + if os.name == 'nt': + extlist.skip_if_missing = True + extlist.extension('storage_sources', self.ss_name + config) + + # Wrapper around session.alter call + def alter(self, uri, alter_param): + # Tiered storage does not fully support alter operation. FIXME WT-9027. + try: + self.session.alter(uri, alter_param) + except BaseException as err: + if self.is_tiered_scenario() and str(err) == 'Operation not supported': + self.skipTest('Tiered storage does not fully support alter operation.') + else: + raise diff --git a/src/third_party/wiredtiger/test/suite/test_alter01.py b/src/third_party/wiredtiger/test/suite/test_alter01.py index d99733f9a12..a6caba59dd6 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter01.py +++ b/src/third_party/wiredtiger/test/suite/test_alter01.py @@ -28,13 +28,15 @@ import wttest from wtscenario import make_scenarios +from helper_tiered import TieredConfigMixin, tiered_storage_sources # test_alter01.py # Smoke-test the session alter operations. -class test_alter01(wttest.WiredTigerTestCase): +class test_alter01(TieredConfigMixin, wttest.WiredTigerTestCase): name = "alter01" entries = 100 - # Settings for access_pattern_hint + + # Data source types types = [ ('file', dict(uri='file:', use_cg=False, use_index=False)), ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), @@ -42,6 +44,8 @@ class test_alter01(wttest.WiredTigerTestCase): ('table-index', dict(uri='table:', use_cg=False, use_index=True)), ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), ] + + # Settings for access_pattern_hint hints = [ ('default', dict(acreate='')), ('none', dict(acreate='none')), @@ -49,6 +53,7 @@ class test_alter01(wttest.WiredTigerTestCase): ('sequential', dict(acreate='sequential')), ] access_alter=('', 'none', 'random', 'sequential') + # Settings for cache_resident resid = [ ('default', dict(ccreate='')), @@ -60,7 +65,9 @@ class test_alter01(wttest.WiredTigerTestCase): ('reopen', dict(reopen=True)), ] cache_alter=('', 'false', 'true') - scenarios = make_scenarios(types, hints, resid, reopen) + + # Build all scenarios + scenarios = make_scenarios(tiered_storage_sources, types, hints, resid, reopen) def verify_metadata(self, metastr): if metastr == '': @@ -87,6 +94,9 @@ class test_alter01(wttest.WiredTigerTestCase): # Alter: Change the access pattern hint after creation def test_alter01_access(self): + if self.is_tiered_scenario() and (self.uri == 'lsm:' or self.uri == 'file:'): + self.skipTest('Tiered storage does not support LSM or file URIs.') + uri = self.uri + self.name create_params = 'key_format=i,value_format=i,' complex_params = '' @@ -153,7 +163,7 @@ class test_alter01(wttest.WiredTigerTestCase): cache_str = 'cache_resident=%s' % c alter_param += ',%s' % cache_str if alter_param != '': - self.session.alter(uri, alter_param) + self.alter(uri, alter_param) if self.reopen: self.reopen_conn() special = self.use_cg or self.use_index @@ -161,7 +171,7 @@ class test_alter01(wttest.WiredTigerTestCase): self.verify_metadata(access_str) self.verify_metadata(cache_str) else: - self.session.alter(suburi, alter_param) + self.alter(suburi, alter_param) self.verify_metadata(access_str) self.verify_metadata(cache_str) diff --git a/src/third_party/wiredtiger/test/suite/test_alter02.py b/src/third_party/wiredtiger/test/suite/test_alter02.py index e01d466f156..ba7fcf19856 100755 --- a/src/third_party/wiredtiger/test/suite/test_alter02.py +++ b/src/third_party/wiredtiger/test/suite/test_alter02.py @@ -26,12 +26,13 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest +import sys, wiredtiger, wttest +from helper_tiered import TieredConfigMixin, tiered_storage_sources from wtscenario import make_scenarios # test_alter02.py # Smoke-test the session alter operations. -class test_alter02(wttest.WiredTigerTestCase): +class test_alter02(TieredConfigMixin, wttest.WiredTigerTestCase): entries = 500 # Binary values. value = u'\u0001\u0002abcd\u0003\u0004' @@ -63,7 +64,7 @@ class test_alter02(wttest.WiredTigerTestCase): ('no-reopen', dict(reopen=False)), ('reopen', dict(reopen=True)), ] - scenarios = make_scenarios(conn_log, types, tables, reopen) + scenarios = make_scenarios(tiered_storage_sources, conn_log, types, tables, reopen) # This test varies the log setting. Override the standard methods. def setUpConnectionOpen(self, dir): @@ -72,8 +73,15 @@ class test_alter02(wttest.WiredTigerTestCase): return None def ConnectionOpen(self): self.home = '.' + + tiered_config = self.conn_config() + tiered_config += self.extensionsConfig() + # In case the open starts additional threads, flush first to avoid confusion. + sys.stdout.flush() conn_params = 'create,log=(file_max=100K,remove=false,%s)' % self.uselog + if tiered_config != '': + conn_params += ',' + tiered_config try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) @@ -129,6 +137,9 @@ class test_alter02(wttest.WiredTigerTestCase): # Alter: Change the log setting after creation def test_alter02_log(self): + if self.is_tiered_scenario() and (self.uri == 'lsm:' or self.uri == 'file:'): + self.skipTest('Tiered storage does not support LSM or file URIs.') + uri = self.uri + self.name create_params = 'key_format=i,value_format=S,' complex_params = '' @@ -208,9 +219,9 @@ class test_alter02(wttest.WiredTigerTestCase): self.conn.close() self.ConnectionOpen() - self.session.alter(uri, alter_param) + self.alter(uri, alter_param) if special: - self.session.alter(suburi, alter_param) + self.alter(suburi, alter_param) self.verify_metadata(log_str) # Put some more data in table. c = self.session.open_cursor(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_alter03.py b/src/third_party/wiredtiger/test/suite/test_alter03.py index 73c55872a2a..fa652c67522 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter03.py +++ b/src/third_party/wiredtiger/test/suite/test_alter03.py @@ -27,12 +27,17 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from helper_tiered import TieredConfigMixin, tiered_storage_sources +from wtscenario import make_scenarios # test_alter03.py # Check if app_metadata can be altered. -class test_alter03(wttest.WiredTigerTestCase): +class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase): name = "alter03" + # Build all scenarios + scenarios = make_scenarios(tiered_storage_sources) + def verify_metadata(self, table_metastr, lsm_metastr, file_metastr): c = self.session.open_cursor('metadata:', None, None) @@ -55,7 +60,15 @@ class test_alter03(wttest.WiredTigerTestCase): if file_metastr != '': # We must find a file type entry for the object and its value # should contain the provided file meta string. - c.set_key('file:' + self.name + '.wt') + if self.is_tiered_scenario(): + c.set_key('file:' + self.name + '-0000000001.wtobj') + + # Removing quotes wrapping app metadata value just to make the test pass. + # FIXME: WT-9036 + file_metastr = 'app_metadata=meta_data_1,' + else: + c.set_key('file:' + self.name + '.wt') + self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND) value = c.get_value() self.assertTrue(value.find(file_metastr) != -1) @@ -81,16 +94,16 @@ class test_alter03(wttest.WiredTigerTestCase): self.verify_metadata(app_meta_orig, '', app_meta_orig) # Alter app metadata and verify - self.session.alter(uri, 'app_metadata="meta_data_2",') + self.alter(uri, 'app_metadata="meta_data_2",') self.verify_metadata('app_metadata="meta_data_2",', '', 'app_metadata="meta_data_2",') # Alter app metadata, explicitly asking for exclusive access and verify - self.session.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,') + self.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,') self.verify_metadata('app_metadata="meta_data_3",', '', 'app_metadata="meta_data_3",') # Alter app metadata without taking exclusive lock and verify that only # table object gets modified - self.session.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,') + self.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,') self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') # Open a cursor, insert some data and try to alter with session open. @@ -100,15 +113,15 @@ class test_alter03(wttest.WiredTigerTestCase): c2[k+1] = 2 self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.alter(uri, 'app_metadata="meta_data_5",')) + lambda: self.alter(uri, 'app_metadata="meta_data_5",')) self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.alter(uri, + lambda: self.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_5",')) self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",') - self.session.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,') + self.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,') self.verify_metadata('app_metadata="meta_data_5",', '', 'app_metadata="meta_data_3",') c2.close() @@ -120,6 +133,9 @@ class test_alter03(wttest.WiredTigerTestCase): # Alter LSM: A non exclusive alter should not be allowed def test_alter03_lsm_app_metadata(self): + if self.is_tiered_scenario(): + self.skipTest('Tiered storage does not support LSM.') + uri = "lsm:" + self.name create_params = 'key_format=i,value_format=i,' app_meta_orig = 'app_metadata="meta_data_1",' @@ -128,17 +144,17 @@ class test_alter03(wttest.WiredTigerTestCase): # Try to alter app metadata without exclusive access and verify self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.alter(uri, + lambda: self.alter(uri, 'exclusive_refreshed=false,app_metadata="meta_data_2",'), '/is applicable only on simple tables/') self.verify_metadata('', 'app_metadata="meta_data_1",', '') # Alter app metadata, explicitly asking for exclusive access and verify - self.session.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",') + self.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",') self.verify_metadata('', 'app_metadata="meta_data_2",', '') # Alter app metadata and verify - self.session.alter(uri, 'app_metadata="meta_data_3",') + self.alter(uri, 'app_metadata="meta_data_3",') self.verify_metadata('', 'app_metadata="meta_data_3",', '') if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_alter04.py b/src/third_party/wiredtiger/test/suite/test_alter04.py index d2d78acf9f6..8c7f87687ee 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter04.py +++ b/src/third_party/wiredtiger/test/suite/test_alter04.py @@ -27,12 +27,13 @@ # OTHER DEALINGS IN THE SOFTWARE. import wttest +from helper_tiered import TieredConfigMixin, tiered_storage_sources from wtscenario import make_scenarios # test_alter04.py # Smoke-test the session alter operations. # This test confirms os_cache_dirty_max and os_cache_max. -class test_alter04(wttest.WiredTigerTestCase): +class test_alter04(TieredConfigMixin, wttest.WiredTigerTestCase): name = "alter04" entries = 100 cache_alter=('1M', '100K') @@ -57,7 +58,7 @@ class test_alter04(wttest.WiredTigerTestCase): ('cache', dict(setting='os_cache_max')), ('cache_dirty', dict(setting='os_cache_dirty_max')), ] - scenarios = make_scenarios(types, sizes, reopen, settings) + scenarios = make_scenarios(tiered_storage_sources, types, sizes, reopen, settings) def verify_metadata(self, metastr): if metastr == '': @@ -84,6 +85,9 @@ class test_alter04(wttest.WiredTigerTestCase): # Alter: Change the setting after creation def test_alter04_cache(self): + if self.is_tiered_scenario() and (self.uri == 'lsm:' or self.uri == 'file:'): + self.skipTest('Tiered storage does not support LSM or file URIs.') + uri = self.uri + self.name create_params = 'key_format=i,value_format=i,' complex_params = '' @@ -129,14 +133,14 @@ class test_alter04(wttest.WiredTigerTestCase): # for all allowed settings. for a in self.cache_alter: alter_param = '%s=%s' % (self.setting, a) - self.session.alter(uri, alter_param) + self.alter(uri, alter_param) if self.reopen: self.reopen_conn() special = self.use_cg or self.use_index if not special: self.verify_metadata(alter_param) else: - self.session.alter(suburi, alter_param) + self.alter(suburi, alter_param) self.verify_metadata(alter_param) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_alter05.py b/src/third_party/wiredtiger/test/suite/test_alter05.py index 2ab67542e4c..6329dbe83ec 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter05.py +++ b/src/third_party/wiredtiger/test/suite/test_alter05.py @@ -27,14 +27,24 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from wiredtiger import stat +from helper_tiered import TieredConfigMixin, tiered_storage_sources +from wtscenario import make_scenarios # test_alter05.py # Check the alter command succeeds even if the file is modified. -class test_alter05(wttest.WiredTigerTestCase): - conn_config = "statistics=(all)" +class test_alter05(TieredConfigMixin, wttest.WiredTigerTestCase): name = "alter05" + scenarios = make_scenarios(tiered_storage_sources) + + # Setup custom connection config. + def conn_config(self): + conf = self.tiered_conn_config() + if conf != '': + conf += ',' + conf += 'statistics=(all)' + return conf + def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') val = stat_cursor[stat][2] @@ -76,13 +86,13 @@ class test_alter05(wttest.WiredTigerTestCase): self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) c.close() - prev_alter_checkpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint) + prev_alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint) # Alter the table and verify. - self.session.alter(uri, 'log=(enabled=false)') + self.alter(uri, 'log=(enabled=false)') self.verify_metadata('log=(enabled=false)') - alter_checkpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint) + alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint) self.assertEqual(prev_alter_checkpoints + 1, alter_checkpoints) prev_alter_checkpoints = alter_checkpoints @@ -94,10 +104,10 @@ class test_alter05(wttest.WiredTigerTestCase): self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) self.assertRaisesException(wiredtiger.WiredTigerError, - lambda: self.session.alter(uri, 'log=(enabled=true)')) + lambda: self.alter(uri, 'log=(enabled=true)')) self.verify_metadata('log=(enabled=false)') - alter_checkpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint) + alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint) self.assertEqual(prev_alter_checkpoints + 1, alter_checkpoints) if __name__ == '__main__': -- cgit v1.2.1