Import wiredtiger: 973bff41fecccf49b1dc2542a408d239857edfa4 from branch mongodb-master

ref: bf2d3db88d..973bff41fe
for: 6.1.0-rc0

WT-9027       Implement support for WT_SESSION::alter for tiered storage

10 files changed, 213 insertions, 41 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 868446bebaf..7653ce1911e 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "bf2d3db88d2b8513adcc55546f36bbd1de00aae1"
+    "commit": "973bff41fecccf49b1dc2542a408d239857edfa4"
 }
diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c
index 3786a719c17..f44715ae25f 100644
--- a/src/third_party/wiredtiger/src/schema/schema_alter.c
+++ b/src/third_party/wiredtiger/src/schema/schema_alter.c
@@ -79,6 +79,26 @@ __alter_file(WT_SESSION_IMPL *session, const char *newcfg[])
 }
 
 /*
+ * __alter_tier --
+ *     Alter a tier.
+ */
+static int
+__alter_tier(WT_SESSION_IMPL *session, const char *newcfg[])
+{
+    const char *uri;
+
+    /*
+     * We know that we have exclusive access to the tier. So it will be closed after we're done with
+     * it and the next open will see the updated metadata.
+     */
+    uri = session->dhandle->name;
+    if (!WT_PREFIX_MATCH(uri, "tier:"))
+        return (__wt_unexpected_object_type(session, uri, "tier:"));
+
+    return (__alter_apply(session, uri, newcfg, WT_CONFIG_BASE(session, tier_meta)));
+}
+
+/*
  * __alter_object --
  *     Alter a tiered object. There are no object dhandles.
  */
@@ -92,6 +112,173 @@ __alter_object(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[])
 }
 
 /*
+ * __alter_get_object_id_range --
+ *     Get current and oldest object IDs for the tiered object.
+ */
+static int
+__alter_get_object_id_range(WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *uri,
+  uint32_t *current_idp, uint32_t *oldest_idp)
+{
+    WT_CONFIG_ITEM cval;
+    WT_DECL_RET;
+    uint32_t current_id, oldest_id;
+    char *value;
+
+    *current_idp = *oldest_idp = WT_TIERED_OBJECTID_NONE;
+    value = NULL;
+    /*
+     * First try to get oldest and current object IDs from the tiered object. If it's not
+     * initialized, get this information from the metadata.
+     */
+    current_id = tiered->current_id;
+    oldest_id = tiered->oldest_id;
+    if (current_id == WT_TIERED_OBJECTID_NONE) {
+        WT_RET(__wt_metadata_search(session, uri, &value));
+
+        WT_ERR(__wt_config_getones(session, value, "oldest", &cval));
+        oldest_id = (uint32_t)cval.val;
+
+        WT_ERR(__wt_config_getones(session, value, "last", &cval));
+        current_id = (uint32_t)cval.val;
+    }
+
+    *current_idp = current_id;
+    *oldest_idp = oldest_id;
+err:
+    __wt_free(session, value);
+
+    return ret;
+}
+
+/*
+ * __alter_objects --
+ *     Alter all objects in the oldest-current range.
+ */
+static int
+__alter_objects(WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *newcfg[],
+  uint32_t current_id, uint32_t oldest_id)
+{
+    WT_DECL_RET;
+    uint32_t object_id;
+    char *value;
+    const char *name;
+
+    value = NULL;
+    name = NULL;
+
+    /*
+     * Iterate over all objects in the range oldest-current object IDs and alter metadata of each of
+     * them.
+     */
+    for (object_id = oldest_id; object_id < current_id; object_id++) {
+        WT_ERR(__wt_tiered_name(session, &tiered->iface, object_id, WT_TIERED_NAME_OBJECT, &name));
+
+        /*
+         * Check if this object present in the metadata. Skip it if not found. It is expected for
+         * the range of objects to be not contiguous, because some objects may have been deleted.
+         */
+        ret = __wt_metadata_search(session, name, &value);
+        __wt_free(session, value);
+        WT_ERR_NOTFOUND_OK(ret, true);
+        if (ret == 0) {
+            WT_WITH_DHANDLE(session, NULL, ret = __schema_alter(session, name, newcfg));
+            WT_ERR(ret);
+        }
+        __wt_free(session, name);
+    }
+err:
+    __wt_free(session, name);
+
+    return (ret);
+}
+
+/*
+ * __alter_tiered --
+ *     Alter a tiered metadata.
+ */
+static int
+__alter_tiered(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[], uint32_t flags)
+{
+    WT_DATA_HANDLE *dhandle;
+    WT_DECL_RET;
+    WT_TIERED *tiered;
+    uint32_t current_id, name_flag, oldest_id;
+    u_int i;
+    char *value;
+    const char *name;
+
+    dhandle = NULL;
+    value = NULL;
+    name = NULL;
+
+    if (!WT_PREFIX_MATCH(uri, "tiered:"))
+        return (__wt_unexpected_object_type(session, uri, "tiered:"));
+
+    /*
+     * If the operation requires exclusive access, close any open handles, including checkpoints.
+     */
+    if (FLD_ISSET(flags, WT_DHANDLE_EXCLUSIVE)) {
+        WT_WITH_HANDLE_LIST_WRITE_LOCK(
+          session, ret = __wt_conn_dhandle_close_all(session, uri, false, false));
+        WT_RET(ret);
+    }
+
+    WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, flags));
+    tiered = (WT_TIERED *)session->dhandle;
+
+    WT_ERR(__alter_get_object_id_range(session, tiered, uri, &current_id, &oldest_id));
+
+    /* Alter each tier. */
+    for (i = 0; i < WT_TIERED_MAX_TIERS; i++) {
+        dhandle = tiered->tiers[i].tier;
+        if (dhandle == NULL) {
+            /*
+             * Tiers may not be initialized because we open tiered handle with lock only flag. In
+             * this case we need to find the names of the tiers manually.
+             */
+            if (i == WT_TIERED_INDEX_LOCAL)
+                name_flag = WT_TIERED_NAME_LOCAL;
+            else if (i == WT_TIERED_INDEX_SHARED)
+                name_flag = WT_TIERED_NAME_SHARED;
+            else
+                continue;
+
+            WT_ERR(__wt_tiered_name(session, &tiered->iface, current_id, name_flag, &name));
+
+            /* Check if metadata has entry for this tier. */
+            ret = __wt_metadata_search(session, name, &value);
+            __wt_free(session, value);
+            WT_ERR_NOTFOUND_OK(ret, true);
+            if (ret != 0) {
+                /*
+                 * Not found in the metadata. Skip it. This is expected, for instance, in the
+                 * scenario when tier hasn't been flushed and there's no shared tier yet.
+                 */
+                __wt_free(session, name);
+                continue;
+            }
+        }
+
+        WT_WITH_DHANDLE(session, NULL,
+          ret = __schema_alter(session, name == NULL ? dhandle->name : name, newcfg));
+        __wt_free(session, name);
+        WT_ERR(ret);
+    }
+
+    /* Alter all objects. */
+    WT_ERR(__alter_objects(session, tiered, newcfg, current_id, oldest_id));
+
+    /* Apply change to the tiered metadata. */
+    WT_ERR(__alter_apply(session, uri, newcfg, WT_CONFIG_BASE(session, tiered_meta)));
+
+err:
+    WT_TRET(__wt_session_release_dhandle(session));
+    __wt_free(session, name);
+
+    return (ret);
+}
+
+/*
  * __alter_tree --
  *     Alter an index or colgroup reference.
  */
@@ -236,9 +423,10 @@ __schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[])
         return (__alter_object(session, uri, newcfg));
     if (WT_PREFIX_MATCH(uri, "table:"))
         return (__alter_table(session, uri, newcfg, exclusive_refreshed));
+    if (WT_PREFIX_MATCH(uri, "tier:"))
+        return (__wt_exclusive_handle_operation(session, uri, __alter_tier, newcfg, flags));
     if (WT_PREFIX_MATCH(uri, "tiered:"))
-        return (__wt_schema_tiered_worker(session, uri, __alter_file, NULL, newcfg, flags));
-
+        return (__alter_tiered(session, uri, newcfg, flags));
     return (__wt_bad_object_type(session, uri));
 }
 
diff --git a/src/third_party/wiredtiger/src/schema/schema_worker.c b/src/third_party/wiredtiger/src/schema/schema_worker.c
index 3ad0ef19a9b..6c3329ee0f1 100644
--- a/src/third_party/wiredtiger/src/schema/schema_worker.c
+++ b/src/third_party/wiredtiger/src/schema/schema_worker.c
@@ -49,13 +49,6 @@ __wt_schema_tiered_worker(WT_SESSION_IMPL *session, const char *uri,
     WT_TIERED *tiered;
     u_int i;
 
-    /*
-     * If this was an alter operation, we need to alter the configuration for the overall tree and
-     * then reread it so it isn't out of date. TODO not yet supported.
-     */
-    if (FLD_ISSET(open_flags, WT_BTREE_ALTER))
-        WT_RET(ENOTSUP);
-
     WT_RET(__wt_session_get_dhandle(session, uri, NULL, NULL, open_flags));
     tiered = (WT_TIERED *)session->dhandle;
 
diff --git a/src/third_party/wiredtiger/test/suite/helper_tiered.py b/src/third_party/wiredtiger/test/suite/helper_tiered.py
index a7b59979921..983274da923 100644
--- a/src/third_party/wiredtiger/test/suite/helper_tiered.py
+++ b/src/third_party/wiredtiger/test/suite/helper_tiered.py
@@ -192,17 +192,6 @@ class TieredConfigMixin:
             extlist.skip_if_missing = True
         extlist.extension('storage_sources', self.ss_name + config)
 
-    # Wrapper around session.alter call
-    def alter(self, uri, alter_param):
-        # Tiered storage does not fully support alter operation. FIXME WT-9027.
-        try:
-            self.session.alter(uri, alter_param)
-        except BaseException as err:
-            if self.is_tiered_scenario() and str(err) == 'Operation not supported':
-                self.skipTest('Tiered storage does not fully support alter operation.')
-            else:
-                raise    
-
     def download_objects(self, bucket_name, prefix):
         import boto3
         # The bucket from the storage source is expected to be a name and a region, separated by a 
diff --git a/src/third_party/wiredtiger/test/suite/test_alter01.py b/src/third_party/wiredtiger/test/suite/test_alter01.py
index 30f6435fc8e..90f62397608 100644
--- a/src/third_party/wiredtiger/test/suite/test_alter01.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter01.py
@@ -164,7 +164,7 @@ class test_alter01(TieredConfigMixin, wttest.WiredTigerTestCase):
                     cache_str = 'cache_resident=%s' % c
                     alter_param += ',%s' % cache_str
                 if alter_param != '':
-                    self.alter(uri, alter_param)
+                    self.session.alter(uri, alter_param)
                     if self.reopen:
                         self.reopen_conn()
                     special = self.use_cg or self.use_index
@@ -172,7 +172,7 @@ class test_alter01(TieredConfigMixin, wttest.WiredTigerTestCase):
                         self.verify_metadata(access_str)
                         self.verify_metadata(cache_str)
                     else:
-                        self.alter(suburi, alter_param)
+                        self.session.alter(suburi, alter_param)
                         self.verify_metadata(access_str)
                         self.verify_metadata(cache_str)
 
diff --git a/src/third_party/wiredtiger/test/suite/test_alter02.py b/src/third_party/wiredtiger/test/suite/test_alter02.py
index 27cb4bd7bb9..13c0755ebca 100755
--- a/src/third_party/wiredtiger/test/suite/test_alter02.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter02.py
@@ -220,9 +220,9 @@ class test_alter02(TieredConfigMixin, wttest.WiredTigerTestCase):
             self.conn.close()
             self.ConnectionOpen()
 
-        self.alter(uri, alter_param)
+        self.session.alter(uri, alter_param)
         if special:
-            self.alter(suburi, alter_param)
+            self.session.alter(suburi, alter_param)
         self.verify_metadata(log_str)
         # Put some more data in table.
         c = self.session.open_cursor(uri, None)
diff --git a/src/third_party/wiredtiger/test/suite/test_alter03.py b/src/third_party/wiredtiger/test/suite/test_alter03.py
index 8c7cff05d12..dcc5c6062a1 100644
--- a/src/third_party/wiredtiger/test/suite/test_alter03.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter03.py
@@ -66,7 +66,8 @@ class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase):
                 
                 # Removing quotes wrapping app metadata value just to make the test pass.
                 # FIXME: WT-9036
-                file_metastr = 'app_metadata=meta_data_1,'
+                if (file_metastr == 'app_metadata="meta_data_1",'):
+                    file_metastr = 'app_metadata=meta_data_1,'
             else:
                 c.set_key('file:' + self.name + '.wt')
 
@@ -95,16 +96,16 @@ class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase):
         self.verify_metadata(app_meta_orig, '', app_meta_orig)
 
         # Alter app metadata and verify
-        self.alter(uri, 'app_metadata="meta_data_2",')
+        self.session.alter(uri, 'app_metadata="meta_data_2",')
         self.verify_metadata('app_metadata="meta_data_2",', '', 'app_metadata="meta_data_2",')
 
         # Alter app metadata, explicitly asking for exclusive access and verify
-        self.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,')
+        self.session.alter(uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,')
         self.verify_metadata('app_metadata="meta_data_3",', '', 'app_metadata="meta_data_3",')
 
         # Alter app metadata without taking exclusive lock and verify that only
         # table object gets modified
-        self.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,')
+        self.session.alter(uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,')
         self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",')
 
         # Open a cursor, insert some data and try to alter with session open.
@@ -114,15 +115,15 @@ class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase):
             c2[k+1] = 2
 
         self.assertRaisesException(wiredtiger.WiredTigerError,
-            lambda: self.alter(uri, 'app_metadata="meta_data_5",'))
+            lambda: self.session.alter(uri, 'app_metadata="meta_data_5",'))
         self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",')
 
         self.assertRaisesException(wiredtiger.WiredTigerError,
-            lambda: self.alter(uri,
+            lambda: self.session.alter(uri,
                 'exclusive_refreshed=true,app_metadata="meta_data_5",'))
         self.verify_metadata('app_metadata="meta_data_4",', '', 'app_metadata="meta_data_3",')
 
-        self.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,')
+        self.session.alter(uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,')
         self.verify_metadata('app_metadata="meta_data_5",', '', 'app_metadata="meta_data_3",')
 
         c2.close()
@@ -145,17 +146,17 @@ class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase):
 
         # Try to alter app metadata without exclusive access and verify
         self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
-            lambda: self.alter(uri,
+            lambda: self.session.alter(uri,
                 'exclusive_refreshed=false,app_metadata="meta_data_2",'),
                 '/is applicable only on simple tables/')
         self.verify_metadata('', 'app_metadata="meta_data_1",', '')
 
         # Alter app metadata, explicitly asking for exclusive access and verify
-        self.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",')
+        self.session.alter(uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",')
         self.verify_metadata('', 'app_metadata="meta_data_2",', '')
 
         # Alter app metadata and verify
-        self.alter(uri, 'app_metadata="meta_data_3",')
+        self.session.alter(uri, 'app_metadata="meta_data_3",')
         self.verify_metadata('', 'app_metadata="meta_data_3",', '')
 
 if __name__ == '__main__':
diff --git a/src/third_party/wiredtiger/test/suite/test_alter04.py b/src/third_party/wiredtiger/test/suite/test_alter04.py
index 8988deb754d..1512c293e8f 100644
--- a/src/third_party/wiredtiger/test/suite/test_alter04.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter04.py
@@ -134,14 +134,15 @@ class test_alter04(TieredConfigMixin, wttest.WiredTigerTestCase):
         # for all allowed settings.
         for a in self.cache_alter:
             alter_param = '%s=%s' % (self.setting, a)
-            self.alter(uri, alter_param)
+            self.session.alter(uri, alter_param)
             if self.reopen:
                 self.reopen_conn()
+
             special = self.use_cg or self.use_index
             if not special:
                 self.verify_metadata(alter_param)
             else:
-                self.alter(suburi, alter_param)
+                self.session.alter(suburi, alter_param)
                 self.verify_metadata(alter_param)
 
 if __name__ == '__main__':
diff --git a/src/third_party/wiredtiger/test/suite/test_alter05.py b/src/third_party/wiredtiger/test/suite/test_alter05.py
index 7d2738d1ef9..47436361e62 100644
--- a/src/third_party/wiredtiger/test/suite/test_alter05.py
+++ b/src/third_party/wiredtiger/test/suite/test_alter05.py
@@ -89,7 +89,7 @@ class test_alter05(TieredConfigMixin, wttest.WiredTigerTestCase):
         prev_alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint)
 
         # Alter the table and verify.
-        self.alter(uri, 'log=(enabled=false)')
+        self.session.alter(uri, 'log=(enabled=false)')
         self.verify_metadata('log=(enabled=false)')
 
         alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint)
@@ -104,7 +104,7 @@ class test_alter05(TieredConfigMixin, wttest.WiredTigerTestCase):
         self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3))
 
         self.assertRaisesException(wiredtiger.WiredTigerError,
-            lambda: self.alter(uri, 'log=(enabled=true)'))
+            lambda: self.session.alter(uri, 'log=(enabled=true)'))
         self.verify_metadata('log=(enabled=false)')
 
         alter_checkpoints = self.get_stat(wiredtiger.stat.conn.session_table_alter_trigger_checkpoint)
diff --git a/src/third_party/wiredtiger/test/suite/test_schema08.py b/src/third_party/wiredtiger/test/suite/test_schema08.py
index 1ff529a0a16..661bbd85d09 100644
--- a/src/third_party/wiredtiger/test/suite/test_schema08.py
+++ b/src/third_party/wiredtiger/test/suite/test_schema08.py
@@ -72,9 +72,9 @@ class test_schema08(TieredConfigMixin, wttest.WiredTigerTestCase, suite_subproce
 
     def do_alter(self, uri, suburi):
         alter_param = 'cache_resident=true'
-        self.alter(uri, alter_param)
+        self.session.alter(uri, alter_param)
         if suburi != None:
-            self.alter(suburi, alter_param)
+            self.session.alter(suburi, alter_param)
 
     def do_ops(self, uri, suburi):
         if (self.schema_ops == 'none'):