diff options
author | Chenhao Qu <chenhao.qu@mongodb.com> | 2022-04-27 01:05:52 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-04-27 01:33:53 +0000 |
commit | fb0c71f8ed69b73492c6d4f373a2080de3cdfb89 (patch) | |
tree | ac82e778745acec4192d2a87faec52494ce116d6 | |
parent | b1eb4a1ca94b3bb3cff23f2a9562aa4433cce26a (diff) | |
download | mongo-fb0c71f8ed69b73492c6d4f373a2080de3cdfb89.tar.gz |
Import wiredtiger: a7e6f71c7610ca7df554781c24c0a9fc9beda656 from branch mongodb-master
ref: 406855114b..a7e6f71c76
for: 6.1.0-rc0
WT-9122 Remove support for internal tiered manager thread
15 files changed, 18 insertions, 322 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 13427efc180..00b79e31fc3 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -835,24 +835,6 @@ connection_runtime_config = [ @ref statistics for more information''', type='list', choices=['all', 'cache_walk', 'fast', 'none', 'clear', 'tree_walk']), - Config('tiered_manager', '', r''' - tiered storage manager configuration options''', - type='category', undoc=True, subconfig=[ - Config('threads_max', '8', r''' - maximum number of threads WiredTiger will start to help manage - tiered storage maintenance. Each worker thread uses a session - from the configured session_max''', - min=1, max=20), - Config('threads_min', '1', r''' - minimum number of threads WiredTiger will start to help manage - tiered storage maintenance.''', - min=1, max=20), - Config('wait', '0', r''' - seconds to wait between each periodic housekeeping of - tiered storage. Setting this value above 0 configures periodic - management inside WiredTiger''', - min='0', max='100000'), - ]), Config('timing_stress_for_test', '', r''' enable code that interrupts the usual timing of operations with a goal of uncovering race conditions and unexpected blocking. This option is diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 6e99eb43941..c76bcf6a26f 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "406855114b113a55f3bc3b6f9a8a09b2a08cc328" + "commit": "a7e6f71c7610ca7df554781c24c0a9fc9beda656" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 9e139b32fc4..68be75e5a75 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -111,11 +111,6 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_statistics_log_su {"sources", "list", NULL, NULL, NULL, 0}, {"timestamp", "string", NULL, NULL, NULL, 0}, {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; -static const WT_CONFIG_CHECK confchk_tiered_manager_subconfigs[] = { - {"threads_max", "int", NULL, "min=1,max=20", NULL, 0}, - {"threads_min", "int", NULL, "min=1,max=20", NULL, 0}, - {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; - static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_tiered_storage_subconfigs[] = { {"local_retention", "int", NULL, "min=0,max=10000", NULL, 0}, {"object_target_size", "int", NULL, "min=100K,max=10TB", NULL, 0}, @@ -155,7 +150,6 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5}, - {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, {"tiered_storage", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_tiered_storage_subconfigs, 2}, {"timing_stress_for_test", "list", NULL, @@ -895,7 +889,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { "\"clear\",\"tree_walk\"]", NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, - {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 7}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," @@ -978,7 +971,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { "\"clear\",\"tree_walk\"]", NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, - {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 7}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," @@ -1059,7 +1051,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { "\"clear\",\"tree_walk\"]", NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, - {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 7}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," @@ -1138,7 +1129,6 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { "\"clear\",\"tree_walk\"]", NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, - {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 7}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," @@ -1212,10 +1202,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "path=\".\"),shared_cache=(chunk=10MB,name=,quota=0,reserve=0," "size=500MB),statistics=none,statistics_log=(json=false," "on_close=false,sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "tiered_manager=(threads_max=8,threads_min=1,wait=0)," "tiered_storage=(local_retention=300,object_target_size=10M)," "timing_stress_for_test=,verbose=[]", - confchk_WT_CONNECTION_reconfigure, 31}, + confchk_WT_CONNECTION_reconfigure, 30}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", "durable_timestamp=,force=false,oldest_timestamp=," @@ -1499,14 +1488,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,bucket=,bucket_prefix=," + ",wait=0),tiered_storage=(auth_token=,bucket=,bucket_prefix=," "cache_directory=,local_retention=300,name=," "object_target_size=10M),timing_stress_for_test=," "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=[]," "verify_metadata=false,write_through=", - confchk_wiredtiger_open, 59}, + confchk_wiredtiger_open, 58}, {"wiredtiger_open_all", "backup_restore_target=," "block_cache=(blkcache_eviction_aggression=1800," @@ -1540,14 +1528,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,bucket=,bucket_prefix=," + ",wait=0),tiered_storage=(auth_token=,bucket=,bucket_prefix=," "cache_directory=,local_retention=300,name=," "object_target_size=10M),timing_stress_for_test=," "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=[]," "verify_metadata=false,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 60}, + confchk_wiredtiger_open_all, 59}, {"wiredtiger_open_basecfg", "backup_restore_target=," "block_cache=(blkcache_eviction_aggression=1800," @@ -1580,13 +1567,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,bucket=,bucket_prefix=," + ",wait=0),tiered_storage=(auth_token=,bucket=,bucket_prefix=," "cache_directory=,local_retention=300,name=," "object_target_size=10M),timing_stress_for_test=," "transaction_sync=(enabled=false,method=fsync),verbose=[]," "verify_metadata=false,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 54}, + confchk_wiredtiger_open_basecfg, 53}, {"wiredtiger_open_usercfg", "backup_restore_target=," "block_cache=(blkcache_eviction_aggression=1800," @@ -1619,13 +1605,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," - "tiered_storage=(auth_token=,bucket=,bucket_prefix=," + ",wait=0),tiered_storage=(auth_token=,bucket=,bucket_prefix=," "cache_directory=,local_retention=300,name=," "object_target_size=10M),timing_stress_for_test=," "transaction_sync=(enabled=false,method=fsync),verbose=[]," "verify_metadata=false,write_through=", - confchk_wiredtiger_open_usercfg, 53}, + confchk_wiredtiger_open_usercfg, 52}, {NULL, NULL, NULL, 0}}; int diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 5b89817c9b6..4bd5c4554e2 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -212,7 +212,7 @@ __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) * can know if statistics are enabled or not. */ WT_RET(__wt_statlog_create(session, cfg)); - WT_RET(__wt_tiered_storage_create(session, cfg)); + WT_RET(__wt_tiered_storage_create(session)); WT_RET(__wt_logmgr_create(session)); /* diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c index 446824f31e8..a11935a2772 100644 --- a/src/third_party/wiredtiger/src/conn/conn_tiered.c +++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c @@ -512,10 +512,6 @@ __wt_flush_tier(WT_SESSION_IMPL *session, const char *config) conn = S2C(session); WT_STAT_CONN_INCR(session, flush_tier); - if (FLD_ISSET(conn->server_flags, WT_CONN_SERVER_TIERED_MGR)) - WT_RET_MSG( - session, EINVAL, "Cannot call flush_tier when storage manager thread is configured"); - flags = 0; cfg[0] = WT_CONFIG_BASE(session, WT_SESSION_flush_tier); cfg[1] = (char *)config; @@ -573,41 +569,6 @@ err: } /* - * __tiered_manager_config -- - * Parse and setup the storage manager options. - */ -static int -__tiered_manager_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) -{ - WT_CONFIG_ITEM cval; - WT_CONNECTION_IMPL *conn; - WT_TIERED_MANAGER *mgr; - - conn = S2C(session); - mgr = &conn->tiered_mgr; - - /* Only start the server if wait time is non-zero */ - WT_RET(__wt_config_gets(session, cfg, "tiered_manager.wait", &cval)); - mgr->wait_usecs = (uint64_t)cval.val * WT_MILLION; - if (runp != NULL) - *runp = mgr->wait_usecs != 0; - - WT_RET(__wt_config_gets(session, cfg, "tiered_manager.threads_max", &cval)); - if (cval.val > WT_TIERED_MAX_WORKERS) - WT_RET_MSG(session, EINVAL, "Maximum storage workers of %" PRIu32 " larger than %d", - (uint32_t)cval.val, WT_TIERED_MAX_WORKERS); - mgr->workers_max = (uint32_t)cval.val; - - WT_RET(__wt_config_gets(session, cfg, "tiered_manager.threads_min", &cval)); - if (cval.val < WT_TIERED_MIN_WORKERS) - WT_RET_MSG(session, EINVAL, "Minimum storage workers of %" PRIu32 " less than %d", - (uint32_t)cval.val, WT_TIERED_MIN_WORKERS); - mgr->workers_min = (uint32_t)cval.val; - WT_ASSERT(session, mgr->workers_min <= mgr->workers_max); - return (0); -} - -/* * __tiered_server -- * The tiered storage server thread. */ @@ -666,106 +627,16 @@ err: } /* - * __tiered_mgr_run_chk -- - * Check to decide if the tiered storage manager should continue running. - */ -static bool -__tiered_mgr_run_chk(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - - conn = S2C(session); - return ((FLD_ISSET(conn->server_flags, WT_CONN_SERVER_TIERED_MGR)) && - !F_ISSET(&conn->tiered_mgr, WT_TIERED_MANAGER_SHUTDOWN)); -} - -/* - * __tiered_mgr_server -- - * The tiered storage manager thread. - */ -static WT_THREAD_RET -__tiered_mgr_server(void *arg) -{ - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - WT_ITEM path, tmp; - WT_SESSION_IMPL *session; - WT_TIERED_MANAGER *mgr; - const char *cfg[2]; - - session = arg; - conn = S2C(session); - mgr = &conn->tiered_mgr; - - WT_CLEAR(path); - WT_CLEAR(tmp); - cfg[0] = "timeout=0"; - cfg[1] = NULL; - - for (;;) { - /* Wait until the next event. */ - __wt_cond_wait(session, conn->tiered_mgr_cond, mgr->wait_usecs, __tiered_mgr_run_chk); - - /* Check if we're quitting or being reconfigured. */ - if (!__tiered_mgr_run_chk(session)) - break; - - /* - * Here is where we do work. Work we expect to do: - */ - WT_WITH_CHECKPOINT_LOCK( - session, WT_WITH_SCHEMA_LOCK(session, ret = __flush_tier_once(session, 0))); - WT_ERR(ret); - if (ret == 0) - WT_ERR(__flush_tier_wait(session, cfg)); - WT_ERR(__tier_storage_remove(session, false)); - } - - if (0) { -err: - WT_IGNORE_RET(__wt_panic(session, ret, "storage server error")); - } - __wt_buf_free(session, &path); - __wt_buf_free(session, &tmp); - return (WT_THREAD_RET_VALUE); -} -/* - * __tiered_mgr_start -- - * Start the tiered manager flush thread. - */ -static int -__tiered_mgr_start(WT_CONNECTION_IMPL *conn) -{ - WT_SESSION_IMPL *session; - - FLD_SET(conn->server_flags, WT_CONN_SERVER_TIERED_MGR); - WT_RET(__wt_open_internal_session( - conn, "storage-mgr-server", false, 0, 0, &conn->tiered_mgr_session)); - session = conn->tiered_mgr_session; - - WT_RET(__wt_cond_alloc(session, "storage server", &conn->tiered_mgr_cond)); - - /* Start the thread. */ - WT_RET(__wt_thread_create(session, &conn->tiered_mgr_tid, __tiered_mgr_server, session)); - conn->tiered_mgr_tid_set = true; - return (0); -} - -/* * __wt_tiered_storage_create -- * Start the tiered storage subsystem. */ int -__wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_tiered_storage_create(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; - bool start; conn = S2C(session); - start = false; - - WT_RET(__tiered_manager_config(session, cfg, &start)); /* Start the internal thread. */ WT_ERR(__wt_cond_alloc(session, "flush tier", &conn->flush_cond)); @@ -785,10 +656,6 @@ __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_thread_create(session, &conn->tiered_tid, __tiered_server, session)); conn->tiered_tid_set = true; - /* After starting non-configurable threads, start the tiered manager if needed. */ - if (start) - WT_ERR(__tiered_mgr_start(conn)); - if (0) { err: FLD_CLR(conn->server_flags, WT_CONN_SERVER_TIERED); @@ -810,19 +677,6 @@ __wt_tiered_storage_destroy(WT_SESSION_IMPL *session) conn = S2C(session); - FLD_CLR(conn->server_flags, WT_CONN_SERVER_TIERED_MGR); - /* - * Stop the storage manager thread. This must be stopped before the internal thread because it - * could be adding work for the internal thread. So stop it first and the internal thread will - * have the opportunity to drain all work. - */ - if (conn->tiered_mgr_tid_set) { - WT_ASSERT(session, conn->tiered_mgr_cond != NULL); - __wt_cond_signal(session, conn->tiered_mgr_cond); - WT_TRET(__wt_thread_join(session, &conn->tiered_mgr_tid)); - conn->tiered_mgr_tid_set = false; - } - /* * Stop the internal server thread. If there is unfinished work, we will recover it on startup * just as if there had been a system failure. @@ -847,14 +701,8 @@ __wt_tiered_storage_destroy(WT_SESSION_IMPL *session) /* Destroy all condition variables after threads have stopped. */ __wt_cond_destroy(session, &conn->tiered_cond); - __wt_cond_destroy(session, &conn->tiered_mgr_cond); /* The flush condition variable must be last because any internal thread could be using it. */ __wt_cond_destroy(session, &conn->flush_cond); - if (conn->tiered_mgr_session != NULL) { - WT_TRET(__wt_session_close_internal(conn->tiered_mgr_session)); - conn->tiered_mgr_session = NULL; - } - return (ret); } diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index c3506557fab..06af2e550ef 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -450,15 +450,6 @@ struct __wt_connection_impl { uint32_t flush_state; /* State of last flush tier */ wt_timestamp_t flush_ts; /* Timestamp of most recent flush_tier */ - WT_TIERED_MANAGER tiered_mgr; /* Tiered manager thread information */ - WT_SESSION_IMPL *tiered_mgr_session; /* Tiered manager thread session */ - wt_thread_t tiered_mgr_tid; /* Tiered manager thread */ - bool tiered_mgr_tid_set; /* Tiered manager thread set */ - WT_CONDVAR *tiered_mgr_cond; /* Tiered manager wait mutex */ - - uint32_t tiered_threads_max; /* Max tiered threads */ - uint32_t tiered_threads_min; /* Min tiered threads */ - /* AUTOMATIC FLAG VALUE GENERATION START 0 */ #define WT_CONN_LOG_CONFIG_ENABLED 0x001u /* Logging is configured */ #define WT_CONN_LOG_DEBUG_MODE 0x002u /* Debug-mode logging enabled */ @@ -628,7 +619,6 @@ struct __wt_connection_impl { #define WT_CONN_SERVER_STATISTICS 0x10u #define WT_CONN_SERVER_SWEEP 0x20u #define WT_CONN_SERVER_TIERED 0x40u -#define WT_CONN_SERVER_TIERED_MGR 0x80u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t server_flags; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index efc2d55a76f..2df1a3126c2 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1504,7 +1504,7 @@ extern int __wt_tiered_put_flush_finish(WT_SESSION_IMPL *session, WT_TIERED *tie WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_set_metadata(WT_SESSION_IMPL *session, WT_TIERED *tiered, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[]) +extern int __wt_tiered_storage_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_storage_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/tiered.h b/src/third_party/wiredtiger/src/include/tiered.h index 5e3e14006dd..89c088b173e 100644 --- a/src/third_party/wiredtiger/src/include/tiered.h +++ b/src/third_party/wiredtiger/src/include/tiered.h @@ -7,26 +7,6 @@ */ /* - * WT_TIERED_MANAGER -- - * A structure that holds resources used to manage any tiered storage - * for the whole database. - */ -struct __wt_tiered_manager { - uint64_t wait_usecs; /* Wait time period */ - uint32_t workers; /* Current number of workers */ - uint32_t workers_max; - uint32_t workers_min; - -#define WT_TIERED_MAX_WORKERS 20 -#define WT_TIERED_MIN_WORKERS 1 - -/* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_TIERED_MANAGER_SHUTDOWN 0x1u /* Manager has shut down */ - /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ - uint32_t flags; -}; - -/* * Define the maximum number of tiers for convenience. We expect at most two initially. This can * change if more are needed. It is easier to have the array statically allocated initially than * worrying about the memory management. For now also assign types to slots. Local files in slot 0. diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 2d80250f349..53ade6681a2 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -333,8 +333,6 @@ struct __wt_thread_group; typedef struct __wt_thread_group WT_THREAD_GROUP; struct __wt_tiered; typedef struct __wt_tiered WT_TIERED; -struct __wt_tiered_manager; -typedef struct __wt_tiered_manager WT_TIERED_MANAGER; struct __wt_tiered_object; typedef struct __wt_tiered_object WT_TIERED_OBJECT; struct __wt_tiered_tiers; diff --git a/src/third_party/wiredtiger/test/suite/helper_tiered.py b/src/third_party/wiredtiger/test/suite/helper_tiered.py index 14552a4d5b0..636eab86006 100644 --- a/src/third_party/wiredtiger/test/suite/helper_tiered.py +++ b/src/third_party/wiredtiger/test/suite/helper_tiered.py @@ -133,7 +133,7 @@ class TieredConfigMixin: 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ - 'name=%s),tiered_manager=(wait=0)' % self.ss_name + 'name=%s),' % self.ss_name # Load the storage sources extension. def conn_extensions(self, extlist): diff --git a/src/third_party/wiredtiger/test/suite/hook_tiered.py b/src/third_party/wiredtiger/test/suite/hook_tiered.py index a3d922b6120..dcc888cd689 100644 --- a/src/third_party/wiredtiger/test/suite/hook_tiered.py +++ b/src/third_party/wiredtiger/test/suite/hook_tiered.py @@ -76,7 +76,7 @@ def wiredtiger_open_tiered(ignored_self, args): tier_string = ',tiered_storage=(auth_token=%s,' % auth_token + \ 'bucket=%s,' % bucket + \ 'bucket_prefix=%s,' % prefix + \ - 'name=%s),tiered_manager=(wait=0),' % extension_name + \ + 'name=%s),' % extension_name + \ 'extensions=[\"%s\"],' % extension_libs[0] args = list(args) # convert from a readonly tuple to a writeable list diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py index bbaf2078e7d..fc40dca9727 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered02.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py @@ -66,7 +66,7 @@ class test_tiered02(wttest.WiredTigerTestCase): 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ - 'name=%s),tiered_manager=(wait=0)' % self.ss_name + 'name=%s),' % self.ss_name # Load the storage store extension. def conn_extensions(self, extlist): diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py deleted file mode 100644 index 8b638788736..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_tiered05.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -from helper_tiered import generate_s3_prefix, get_auth_token, get_bucket1_name -import os, time, wiredtiger, wttest -from wtscenario import make_scenarios -StorageSource = wiredtiger.StorageSource # easy access to constants - -# test_tiered05.py -# Basic tiered storage API test error for tiered manager and flush_tier. -class test_tiered05(wttest.WiredTigerTestCase): - storage_sources = [ - ('dir_store', dict(auth_token = get_auth_token('dir_store'), - bucket = get_bucket1_name('dir_store'), - bucket_prefix = "pfx_", - ss_name = 'dir_store')), - ('s3', dict(auth_token = get_auth_token('s3_store'), - bucket = get_bucket1_name('s3_store'), - bucket_prefix = generate_s3_prefix(), - ss_name = 's3_store')), - ] - # Make scenarios for different cloud service providers - scenarios = make_scenarios(storage_sources) - - uri = "table:test_tiered05" - wait = 2 - - def conn_extensions(self, extlist): - config = '' - # S3 store is built as an optional loadable extension, not all test environments build S3. - if self.ss_name == 's3_store': - #config = '=(config=\"(verbose=1)\")' - extlist.skip_if_missing = True - #if self.ss_name == 'dir_store': - #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")' - # Windows doesn't support dynamically loaded extension libraries. - if os.name == 'nt': - extlist.skip_if_missing = True - extlist.extension('storage_sources', self.ss_name + config) - - def conn_config(self): - if self.ss_name == 'dir_store' and not os.path.exists(self.bucket): - os.mkdir(self.bucket) - return \ - 'tiered_manager=(wait=%d),' % self.wait + \ - 'tiered_storage=(auth_token=%s,' % self.auth_token + \ - 'bucket=%s,' % self.bucket + \ - 'bucket_prefix=%s,' % self.bucket_prefix + \ - 'name=%s,' % self.ss_name + \ - 'object_target_size=20M)' - - # Test calling the flush_tier API with a tiered manager. Should get an error. - def test_tiered(self): - self.session.create(self.uri, 'key_format=S') - # Allow time for the thread to start up. - time.sleep(self.wait) - msg = "/storage manager thread is configured/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(self.session.flush_tier(None), 0), msg) - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered08.py b/src/third_party/wiredtiger/test/suite/test_tiered08.py index 50b80f6b54e..4e8b1b50520 100644 --- a/src/third_party/wiredtiger/test/suite/test_tiered08.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered08.py @@ -70,7 +70,7 @@ class test_tiered08(wttest.WiredTigerTestCase): 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ - 'name=%s),tiered_manager=(wait=0)' % self.ss_name + 'name=%s),' % self.ss_name # Load the storage store extension. def conn_extensions(self, extlist): diff --git a/src/third_party/wiredtiger/test/suite/test_tiered14.py b/src/third_party/wiredtiger/test/suite/test_tiered14.py index 61e3a54cf91..ab210d0bae2 100644 --- a/src/third_party/wiredtiger/test/suite/test_tiered14.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered14.py @@ -79,7 +79,7 @@ class test_tiered14(wttest.WiredTigerTestCase): 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ - 'name=%s),tiered_manager=(wait=0)' % self.ss_name + 'name=%s),' % self.ss_name # Load the storage store extension. def conn_extensions(self, extlist): |