Import wiredtiger: 930ccee6ffc8f9f1f2ff77e74d1b47be150c2409 from branch mongodb-master

ref: c576dfa728..930ccee6ff
for: 6.0.0-rc0

WT-9026       Add tiered storage scenario to all test_alter python tests

7 files changed, 162 insertions, 34 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index f60840276a0..ea87ab9a76b 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "c576dfa7288511efa2ea82865491ad1f429d3516"
+    "commit": "930ccee6ffc8f9f1f2ff77e74d1b47be150c2409"
 }
diff --git a/src/third_party/wiredtiger/test/suite/helper_tiered.py b/src/third_party/wiredtiger/test/suite/helper_tiered.py
index 64406130d63..bb252f588ea 100644
--- a/src/third_party/wiredtiger/test/suite/helper_tiered.py
+++ b/src/third_party/wiredtiger/test/suite/helper_tiered.py
@@ -90,3 +90,80 @@ def generate_s3_prefix(test_name = ''):
 
     return prefix
 
+# Storage sources.
+tiered_storage_sources = [
+    ('dirstore', dict(is_tiered = True,
+        is_local_storage = True,
+        auth_token = get_auth_token('dir_store'),
+        bucket = get_bucket1_name('dir_store'),
+        bucket_prefix = "pfx_",
+        ss_name = 'dir_store')),
+    ('s3', dict(is_tiered = True,
+        is_local_storage = False,
+        auth_token = get_auth_token('s3_store'),
+        bucket = get_bucket1_name('s3_store'),
+        bucket_prefix = generate_s3_prefix(),
+        ss_name = 's3_store')),
+    ('non_tiered', dict(is_tiered = False)),            
+]
+
+# This mixin class provides tiered storage configuration methods.
+class TieredConfigMixin:
+    # Returns True if the current scenario is tiered.
+    def is_tiered_scenario(self):
+        return hasattr(self, 'is_tiered') and self.is_tiered
+
+    # Setup connection config.
+    def conn_config(self):
+        return self.tiered_conn_config()
+
+    # Setup tiered connection config.
+    def tiered_conn_config(self):
+        # Handle non_tiered storage scenarios.
+        if not self.is_tiered_scenario():
+            return ''
+
+        # Setup directories structure for local tiered storage.
+        if self.is_local_storage and not os.path.exists(self.bucket):
+            os.mkdir(self.bucket)
+
+        # Build tiered storage connection string.
+        return \
+            'debug_mode=(flush_checkpoint=true),' + \
+            'tiered_storage=(auth_token=%s,' % self.auth_token + \
+            'bucket=%s,' % self.bucket + \
+            'bucket_prefix=%s,' % self.bucket_prefix + \
+            'name=%s),tiered_manager=(wait=0)' % self.ss_name
+
+    # Load the storage sources extension.
+    def conn_extensions(self, extlist):
+        return self.tiered_conn_extensions(extlist)
+
+    # Load tiered storage source extension.
+    def tiered_conn_extensions(self, extlist):
+        # Handle non_tiered storage scenarios.
+        if not self.is_tiered_scenario():
+            return ''
+        
+        config = ''
+        # S3 store is built as an optional loadable extension, not all test environments build S3.
+        if not self.is_local_storage:
+            #config = '=(config=\"(verbose=1)\")'
+            extlist.skip_if_missing = True
+        #if self.is_local_storage:
+            #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")'
+        # Windows doesn't support dynamically loaded extension libraries.
+        if os.name == 'nt':
+            extlist.skip_if_missing = True
+        extlist.extension('storage_sources', self.ss_name + config)
+
+    # Wrapper around session.alter call
+    def alter(self, uri, alter_param):
+        # Tiered storage does not fully support alter operation. FIXME WT-9027.
+        try:
+            self.session.alter(uri, alter_param)
+        except BaseException as err:
+            if self.is_tiered_scenario() and str(err) == 'Operation not supported':
+                self.skipTest('Tiered storage does not fully support alter operation.')
+            else:
+                raise    
diff --git a/src/third_party/wiredtiger/test/suite/test_alter01.py b/src/third_party/wiredtiger/test/suite/test_alter01.py
index d99733f9a12..a6caba59dd6 100644
--- a/src/third_party/wiredtiger/test/suite/test_alter01.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter01.py
@@ -28,13 +28,15 @@
 
 import wttest
 from wtscenario import make_scenarios
+from helper_tiered import TieredConfigMixin, tiered_storage_sources
 
 # test_alter01.py
 #    Smoke-test the session alter operations.
-class test_alter01(wttest.WiredTigerTestCase):
+class test_alter01(TieredConfigMixin, wttest.WiredTigerTestCase):
     name = "alter01"
     entries = 100
-    # Settings for access_pattern_hint
+
+    # Data source types
     types = [
         ('file', dict(uri='file:', use_cg=False, use_index=False)),
         ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)),
@@ -42,6 +44,8 @@ class test_alter01(wttest.WiredTigerTestCase):
         ('table-index', dict(uri='table:', use_cg=False, use_index=True)),
         ('table-simple', dict(uri='table:', use_cg=False, use_index=False)),
     ]
+
+    # Settings for access_pattern_hint
     hints = [
         ('default', dict(acreate='')),
         ('none', dict(acreate='none')),
@@ -49,6 +53,7 @@ class test_alter01(wttest.WiredTigerTestCase):
         ('sequential', dict(acreate='sequential')),
     ]
     access_alter=('', 'none', 'random', 'sequential')
+
     # Settings for cache_resident
     resid = [
         ('default', dict(ccreate='')),
@@ -60,7 +65,9 @@ class test_alter01(wttest.WiredTigerTestCase):
         ('reopen', dict(reopen=True)),
     ]
     cache_alter=('', 'false', 'true')
-    scenarios = make_scenarios(types, hints, resid, reopen)
+
+    # Build all scenarios
+    scenarios = make_scenarios(tiered_storage_sources, types, hints, resid, reopen)
 
     def verify_metadata(self, metastr):
         if metastr == '':
@@ -87,6 +94,9 @@ class test_alter01(wttest.WiredTigerTestCase):
 
     # Alter: Change the access pattern hint after creation
     def test_alter01_access(self):
+        if self.is_tiered_scenario() and (self.uri == 'lsm:' or self.uri == 'file:'):
+            self.skipTest('Tiered storage does not support LSM or file URIs.')
+
         uri = self.uri + self.name
         create_params = 'key_format=i,value_format=i,'
         complex_params = ''
@@ -153,7 +163,7 @@ class test_alter01(wttest.WiredTigerTestCase):
                     cache_str = 'cache_resident=%s' % c
                     alter_param += ',%s' % cache_str
                 if alter_param != '':
-                    self.session.alter(uri, alter_param)
+                    self.alter(uri, alter_param)
                     if self.reopen:
                         self.reopen_conn()
                     special = self.use_cg or self.use_index
@@ -161,7 +171,7 @@ class test_alter01(wttest.WiredTigerTestCase):
                         self.verify_metadata(access_str)
                         self.verify_metadata(cache_str)
                     else:
-                        self.session.alter(suburi, alter_param)
+                        self.alter(suburi, alter_param)
                         self.verify_metadata(access_str)
                         self.verify_metadata(cache_str)
 
diff --git a/src/third_party/wiredtiger/test/suite/test_alter02.py b/src/third_party/wiredtiger/test/suite/test_alter02.py
index e01d466f156..ba7fcf19856 100755
--- a/src/third_party/wiredtiger/test/suite/test_alter02.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter02.py
@@ -26,12 +26,13 @@
 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 # OTHER DEALINGS IN THE SOFTWARE.
 
-import wiredtiger, wttest
+import sys, wiredtiger, wttest
+from helper_tiered import TieredConfigMixin, tiered_storage_sources
 from wtscenario import make_scenarios
 
 # test_alter02.py
 #    Smoke-test the session alter operations.
-class test_alter02(wttest.WiredTigerTestCase):
+class test_alter02(TieredConfigMixin, wttest.WiredTigerTestCase):
     entries = 500
     # Binary values.
     value = u'\u0001\u0002abcd\u0003\u0004'
@@ -63,7 +64,7 @@ class test_alter02(wttest.WiredTigerTestCase):
         ('no-reopen', dict(reopen=False)),
         ('reopen', dict(reopen=True)),
     ]
-    scenarios = make_scenarios(conn_log, types, tables, reopen)
+    scenarios = make_scenarios(tiered_storage_sources, conn_log, types, tables, reopen)
 
     # This test varies the log setting.  Override the standard methods.
     def setUpConnectionOpen(self, dir):
@@ -72,8 +73,15 @@ class test_alter02(wttest.WiredTigerTestCase):
         return None
     def ConnectionOpen(self):
         self.home = '.'
+        
+        tiered_config = self.conn_config()
+        tiered_config += self.extensionsConfig()
+        # In case the open starts additional threads, flush first to avoid confusion.
+        sys.stdout.flush()
 
         conn_params = 'create,log=(file_max=100K,remove=false,%s)' % self.uselog
+        if tiered_config != '':
+            conn_params += ',' + tiered_config
 
         try:
             self.conn = wiredtiger.wiredtiger_open(self.home, conn_params)
@@ -129,6 +137,9 @@ class test_alter02(wttest.WiredTigerTestCase):
 
     # Alter: Change the log setting after creation
     def test_alter02_log(self):
+        if self.is_tiered_scenario() and (self.uri == 'lsm:' or self.uri == 'file:'):
+            self.skipTest('Tiered storage does not support LSM or file URIs.')
+        
         uri = self.uri + self.name
         create_params = 'key_format=i,value_format=S,'
         complex_params = ''
@@ -208,9 +219,9 @@ class test_alter02(wttest.WiredTigerTestCase):
             self.conn.close()
             self.ConnectionOpen()
 
-        self.session.alter(uri, alter_param)
+        self.alter(uri, alter_param)
         if special:
-            self.session.alter(suburi, alter_param)
+            self.alter(suburi, alter_param)
         self.verify_metadata(log_str)
         # Put some more data in table.
         c = self.session.open_cursor(uri, None)
diff --git a/src/third_party/wiredtiger/test/suite/test_alter03.py b/src/third_party/wiredtiger/test/suite/test_alter03.py
index 73c55872a2a..fa652c67522 100644
--- a/src/third_party/wiredtiger/test/suite/test_alter03.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter03.py
@@ -27,12 +27,17 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 
 import wiredtiger, wttest
+from helper_tiered import TieredConfigMixin, tiered_storage_sources
+from wtscenario import make_scenarios
 
 # test_alter03.py
 #    Check if app_metadata can be altered.
-class test_alter03(wttest.WiredTigerTestCase):
+class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase):
     name = "alter03"
 
+    # Build all scenarios
+    scenarios = make_scenarios(tiered_storage_sources)
+
     def verify_metadata(self, table_metastr, lsm_metastr, file_metastr):
         c = self.session.open_cursor('metadata:', None, None)
 
@@ -55,7 +60,15 @@ class test_alter03(wttest.WiredTigerTestCase):
         if file_metastr != '':
             # We must find a file type entry for the object and its value
             # should contain the provided file meta string.
-            c.set_key('file:' + self.name + '.wt')
+            if self.is_tiered_scenario():
+                c.set_key('file:' + self.name + '-0000000001.wtobj')
+                
+                # Removing quotes wrapping app metadata value just to make the test pass.
+                # FIXME: WT-9036
+                file_metastr = 'app_metadata=meta_data_1,'
+            else:
+                c.set_key('file:' + self.name + '.wt')
+
             self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND)
             value = c.get_value()
             self.assertTrue(value.find(file_metastr) != -1)
@@ -81,16 +94,16 @@ class test_alter03(wttest.WiredTigerTestCase):
         self.verify_metadata(app_meta_orig, '', app_meta_orig)
 
         # Alter app metadata and verify
-        self.session.alter(uri, 'app_metadata="meta_data_2",')
+        self.alter(uri, 'app_metadata="meta_data_2",')
         self.verify_metadata('app_metadata="meta_data_2",', '', 'app_metadata="meta_data_2",')
 
         # Alter app metadata, explicitly asking for exclusive access and verify
-        self.session.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,')
+        self.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,')
         self.verify_metadata('app_metadata="meta_data_3",', '', 'app_metadata="meta_data_3",')
 
         # Alter app metadata without taking exclusive lock and verify that only
         # table object gets modified
-        self.session.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,')
+        self.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,')
         self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",')
 
         # Open a cursor, insert some data and try to alter with session open.
@@ -100,15 +113,15 @@ class test_alter03(wttest.WiredTigerTestCase):
             c2[k+1] = 2
 
         self.assertRaisesException(wiredtiger.WiredTigerError,
-            lambda: self.session.alter(uri, 'app_metadata="meta_data_5",'))
+            lambda: self.alter(uri, 'app_metadata="meta_data_5",'))
         self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",')
 
         self.assertRaisesException(wiredtiger.WiredTigerError,
-            lambda: self.session.alter(uri,
+            lambda: self.alter(uri,
                 'exclusive_refreshed=true,app_metadata="meta_data_5",'))
         self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",')
 
-        self.session.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,')
+        self.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,')
         self.verify_metadata('app_metadata="meta_data_5",', '', 'app_metadata="meta_data_3",')
 
         c2.close()
@@ -120,6 +133,9 @@ class test_alter03(wttest.WiredTigerTestCase):
 
     # Alter LSM: A non exclusive alter should not be allowed
     def test_alter03_lsm_app_metadata(self):
+        if self.is_tiered_scenario():
+            self.skipTest('Tiered storage does not support LSM.')
+        
         uri = "lsm:" + self.name
         create_params = 'key_format=i,value_format=i,'
         app_meta_orig = 'app_metadata="meta_data_1",'
@@ -128,17 +144,17 @@ class test_alter03(wttest.WiredTigerTestCase):
 
         # Try to alter app metadata without exclusive access and verify
         self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
-            lambda: self.session.alter(uri,
+            lambda: self.alter(uri,
                 'exclusive_refreshed=false,app_metadata="meta_data_2",'),
                 '/is applicable only on simple tables/')
         self.verify_metadata('', 'app_metadata="meta_data_1",', '')
 
         # Alter app metadata, explicitly asking for exclusive access and verify
-        self.session.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",')
+        self.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",')
         self.verify_metadata('', 'app_metadata="meta_data_2",', '')
 
         # Alter app metadata and verify
-        self.session.alter(uri, 'app_metadata="meta_data_3",')
+        self.alter(uri, 'app_metadata="meta_data_3",')
         self.verify_metadata('', 'app_metadata="meta_data_3",', '')
 
 if __name__ == '__main__':
diff --git a/src/third_party/wiredtiger/test/suite/test_alter04.py b/src/third_party/wiredtiger/test/suite/test_alter04.py
index d2d78acf9f6..8c7f87687ee 100644
--- a/src/third_party/wiredtiger/test/suite/test_alter04.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter04.py
@@ -27,12 +27,13 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 
 import wttest
+from helper_tiered import TieredConfigMixin, tiered_storage_sources
 from wtscenario import make_scenarios
 
 # test_alter04.py
 #    Smoke-test the session alter operations.
 #    This test confirms os_cache_dirty_max and os_cache_max.
-class test_alter04(wttest.WiredTigerTestCase):
+class test_alter04(TieredConfigMixin, wttest.WiredTigerTestCase):
     name = "alter04"
     entries = 100
     cache_alter=('1M', '100K')
@@ -57,7 +58,7 @@ class test_alter04(wttest.WiredTigerTestCase):
         ('cache', dict(setting='os_cache_max')),
         ('cache_dirty', dict(setting='os_cache_dirty_max')),
     ]
-    scenarios = make_scenarios(types, sizes, reopen, settings)
+    scenarios = make_scenarios(tiered_storage_sources, types, sizes, reopen, settings)
 
     def verify_metadata(self, metastr):
         if metastr == '':
@@ -84,6 +85,9 @@ class test_alter04(wttest.WiredTigerTestCase):
 
     # Alter: Change the setting after creation
     def test_alter04_cache(self):
+        if self.is_tiered_scenario() and (self.uri == 'lsm:' or self.uri == 'file:'):
+            self.skipTest('Tiered storage does not support LSM or file URIs.')
+        
         uri = self.uri + self.name
         create_params = 'key_format=i,value_format=i,'
         complex_params = ''
@@ -129,14 +133,14 @@ class test_alter04(wttest.WiredTigerTestCase):
         # for all allowed settings.
         for a in self.cache_alter:
             alter_param = '%s=%s' % (self.setting, a)
-            self.session.alter(uri, alter_param)
+            self.alter(uri, alter_param)
             if self.reopen:
                 self.reopen_conn()
             special = self.use_cg or self.use_index
             if not special:
                 self.verify_metadata(alter_param)
             else:
-                self.session.alter(suburi, alter_param)
+                self.alter(suburi, alter_param)
                 self.verify_metadata(alter_param)
 
 if __name__ == '__main__':
diff --git a/src/third_party/wiredtiger/test/suite/test_alter05.py b/src/third_party/wiredtiger/test/suite/test_alter05.py
index 2ab67542e4c..6329dbe83ec 100644
--- a/src/third_party/wiredtiger/test/suite/test_alter05.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter05.py
@@ -27,14 +27,24 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 
 import wiredtiger, wttest
-from wiredtiger import stat
+from helper_tiered import TieredConfigMixin, tiered_storage_sources
+from wtscenario import make_scenarios
 
 # test_alter05.py
 #    Check the alter command succeeds even if the file is modified.
-class test_alter05(wttest.WiredTigerTestCase):
-    conn_config = "statistics=(all)"
+class test_alter05(TieredConfigMixin, wttest.WiredTigerTestCase):
     name = "alter05"
 
+    scenarios = make_scenarios(tiered_storage_sources)
+
+    # Setup custom connection config.
+    def conn_config(self):
+        conf = self.tiered_conn_config()
+        if conf != '':
+            conf += ','
+        conf += 'statistics=(all)'
+        return conf
+
     def get_stat(self, stat):
         stat_cursor = self.session.open_cursor('statistics:')
         val = stat_cursor[stat][2]
@@ -76,13 +86,13 @@ class test_alter05(wttest.WiredTigerTestCase):
         self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2))
         c.close()
 
-        prev_alter_checkpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint)
+        prev_alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint)
 
         # Alter the table and verify.
-        self.session.alter(uri, 'log=(enabled=false)')
+        self.alter(uri, 'log=(enabled=false)')
         self.verify_metadata('log=(enabled=false)')
 
-        alter_checkpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint)
+        alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint)
         self.assertEqual(prev_alter_checkpoints + 1, alter_checkpoints)
         prev_alter_checkpoints = alter_checkpoints
 
@@ -94,10 +104,10 @@ class test_alter05(wttest.WiredTigerTestCase):
         self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3))
 
         self.assertRaisesException(wiredtiger.WiredTigerError,
-            lambda: self.session.alter(uri, 'log=(enabled=true)'))
+            lambda: self.alter(uri, 'log=(enabled=true)'))
         self.verify_metadata('log=(enabled=false)')
 
-        alter_checkpoints = self.get_stat(stat.conn.session_table_alter_trigger_checkpoint)
+        alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint)
         self.assertEqual(prev_alter_checkpoints + 1, alter_checkpoints)
 
 if __name__ == '__main__':