diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-02-17 15:58:51 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-02-17 05:25:49 +0000 |
| commit | 8c2c462fbb802a2093fae9be5e21b4a1d723988b (patch) | |
| tree | 4bf575a32b768669118a25abf4c92cfe62b01f7f | |
| parent | ed6aa0cf8271075bf4e37ce8890d61b7fc089417 (diff) | |
| download | mongo-8c2c462fbb802a2093fae9be5e21b4a1d723988b.tar.gz | |
Import wiredtiger: f90baa428bfefb17afddd8f8aa72e8d34cf761e2 from branch mongodb-5.0
ref: 5641affc86..f90baa428b
for: 4.9.0
WT-6513 Update history store configuration to make overflow keys unlikely
WT-6850 Create and add config parsing for the workload generator class
WT-6950 Create test coverage documentation landing page
WT-7087 API for tiered storage
WT-7211 Added missing return statement in stress testing framework
32 files changed, 1450 insertions, 348 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 74c76adc5c7..c42cf702495 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -721,6 +721,24 @@ connection_runtime_config = [ this will update the value if one is already set''', min='1MB', max='10TB') ]), + Config('tiered_manager', '', r''' + tiered storage manager configuration options''', + type='category', undoc=True, subconfig=[ + Config('threads_max', '8', r''' + maximum number of threads WiredTiger will start to help manage + tiered storage maintenance. Each worker thread uses a session + from the configured session_max''', + min=1, max=20), + Config('threads_min', '1', r''' + minimum number of threads WiredTiger will start to help manage + tiered storage maintenance.''', + min=1, max=20), + Config('wait', '0', r''' + seconds to wait between each periodic housekeeping of + tiered storage. Setting this value above 0 configures periodic + management inside WiredTiger''', + min='0', max='100000'), + ]), Config('statistics', 'none', r''' Maintain database statistics, which may impact performance. Choosing "all" maintains all statistics regardless of cost, @@ -910,6 +928,42 @@ connection_reconfigure_statistics_log_configuration = [ type='category', subconfig= statistics_log_configuration_common) ] + +tiered_storage_configuration_common = [ + Config('auth_token', '', r''' + authentication token string'''), + Config('local_retention', '300', r''' + time in seconds to retain data on tiered storage on the local tier for + faster read access''', + min='0', max='10000'), + Config('object_target_size', '10M', r''' + the approximate size of objects before creating them on the + tiered storage tier''', + min='100K', max='10TB'), +] +connection_reconfigure_tiered_storage_configuration = [ + Config('tiered_storage', '', r''' + enable tiered storage. Enabling tiered storage may use one session from the + configured session_max''', + type='category', subconfig= + tiered_storage_configuration_common) +] +wiredtiger_open_tiered_storage_configuration = [ + Config('tiered_storage', '', r''' + enable tiered storage. Enabling tiered storage may use one session from the + configured session_max''', + type='category', undoc=True, subconfig= + tiered_storage_configuration_common + [ + Config('enabled', 'false', r''' + enable tiered storage subsystem''', + type='boolean'), + Config('name', 'none', r''' + Permitted values are \c "none" + or custom storage name created with + WT_CONNECTION::add_tiered_storage'''), + ]), +] + wiredtiger_open_statistics_log_configuration = [ Config('statistics_log', '', r''' log any statistics the database is configured to maintain, @@ -950,6 +1004,7 @@ wiredtiger_open_common =\ connection_runtime_config +\ wiredtiger_open_compatibility_configuration +\ wiredtiger_open_log_configuration +\ + wiredtiger_open_tiered_storage_configuration +\ wiredtiger_open_statistics_log_configuration + [ Config('buffer_alignment', '-1', r''' in-memory alignment (in bytes) for buffers used for I/O. The @@ -1448,6 +1503,13 @@ methods = { files''', type='boolean'), ]), + +'WT_SESSION.flush_tier' : Method([ + Config('force', 'false', r''' + force sharing of all data''', + type='boolean'), +]), + 'WT_SESSION.strerror' : Method([]), 'WT_SESSION.transaction_sync' : Method([ Config('timeout_ms', '1200000', # !!! Must match WT_SESSION_BG_SYNC_MSEC @@ -1685,6 +1747,7 @@ methods = { connection_reconfigure_compatibility_configuration +\ connection_reconfigure_log_configuration +\ connection_reconfigure_statistics_log_configuration +\ + connection_reconfigure_tiered_storage_configuration +\ connection_runtime_config ), 'WT_CONNECTION.set_file_system' : Method([]), diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 39b85746b6c..35ff295c206 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -74,6 +74,7 @@ src/conn/conn_open.c src/conn/conn_reconfig.c src/conn/conn_stat.c src/conn/conn_sweep.c +src/conn/conn_tiered.c src/cursor/cur_backup.c src/cursor/cur_backup_incr.c src/cursor/cur_bulk.c diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index f9f2b67712f..c5b3fe73c20 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -503,6 +503,7 @@ aspell assertfmt async atomicity +auth autocommit autoconf automake diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 77808db1c9d..97c20322319 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -104,6 +104,10 @@ class SessionOpStat(Stat): prefix = 'session' def __init__(self, name, desc, flags=''): Stat.__init__(self, name, SessionOpStat.prefix, desc, flags) +class StorageStat(Stat): + prefix = 'session' + def __init__(self, name, desc, flags=''): + Stat.__init__(self, name, SessionOpStat.prefix, desc, flags) class ThreadStat(Stat): prefix = 'thread-state' def __init__(self, name, desc, flags=''): @@ -463,6 +467,8 @@ connection_stats = [ # Reconciliation statistics ########################################## RecStat('rec_maximum_seconds', 'maximum seconds spent in a reconciliation call', 'no_clear,no_scale,size'), + RecStat('rec_overflow_key_internal', 'internal-page overflow keys'), + RecStat('rec_overflow_key_leaf', 'leaf-page overflow keys'), RecStat('rec_pages_with_prepare', 'page reconciliation calls that resulted in values with prepared transaction metadata'), RecStat('rec_pages_with_ts', 'page reconciliation calls that resulted in values with timestamps'), RecStat('rec_pages_with_txn', 'page reconciliation calls that resulted in values with transaction ids'), @@ -855,6 +861,12 @@ conn_dsrc_stats = [ RecStat('rec_time_window_stop_txn', 'records written including a stop transaction ID'), ########################################## + # Tiered storage statistics + ########################################## + StorageStat('flush_tier', 'flush_tier operation calls'), + StorageStat('tiered_retention', 'tiered storage local retention time (secs)', 'no_clear,no_scale,size'), + + ########################################## # Transaction statistics ########################################## TxnStat('txn_checkpoint_obsolete_applied', 'transaction checkpoints due to obsolete pages'), diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py index ac46cf55bc2..3fedb10b5d4 100644 --- a/src/third_party/wiredtiger/dist/test_data.py +++ b/src/third_party/wiredtiger/dist/test_data.py @@ -41,16 +41,40 @@ class Config: def __ge__(self, other): return self.name >= other.name + +key_config=[ + Config('key_size', 0, r''' + The size of the keys to be created''', min=0, max=10000), +] + +value_config = [ + Config('value_size', 0, r''' + The size of the values to be created''', min=0, max=10000), +] + +scale_config = [ + Config('collection_count', 1, r''' + The number of colections the workload generator operates over''', min=0, max=200000), + Config('key_count', 0, r''' + The number of keys to be operated on per colection''', min=0, max=1000000), +] + +load_config = key_config + value_config + scale_config + +workload_config = [ + Config('read_threads', 0, r''' + The number of threads performing read operations''', min=0, max=100), + Config('insert_threads', 0, r''' + The number of threads performing insert operations''',min=0, max=20), + Config('insert_config',0, r''' + The definition of the record being inserted''', + subconfig=load_config), + Config('update_threads', 0, r''' + The number of threads performing update operations''',min=0, max=20), + Config('update_config',0,r''', + The definition of the record being updated''', subconfig=load_config) +] + methods = { -'poc_test' : Method([ - Config('collection_count', '1', r''' - the number of collections to create for testing''', - min='1', max='10'), - Config('key_size', '10', r''' - the size of the keys to be created in bytes''', - min='1', max='10000'), - Config('values', 'first', r''' - The value that each key will be populated with, used an example string configuration''', - choices=['first', 'second', 'third']) -]), +'poc_test' : Method(load_config + workload_config), } diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index ab8faa77e69..8139c629724 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "5641affc867b1e150b2528471b4a876ea999b2e7" + "commit": "f90baa428bfefb17afddd8f8aa72e8d34cf761e2" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index bb87d9e3535..d4552f9c711 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -90,6 +90,17 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_statistics_log_su {"sources", "list", NULL, NULL, NULL, 0}, {"timestamp", "string", NULL, NULL, NULL, 0}, {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_tiered_manager_subconfigs[] = { + {"threads_max", "int", NULL, "min=1,max=20", NULL, 0}, + {"threads_min", "int", NULL, "min=1,max=20", NULL, 0}, + {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + +static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_tiered_storage_subconfigs[] = { + {"auth_token", "string", NULL, NULL, NULL, 0}, + {"local_retention", "int", NULL, "min=0,max=10000", NULL, 0}, + {"object_target_size", "int", NULL, "min=100K,max=10TB", NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"cache_max_wait_ms", "int", NULL, "min=0", NULL, 0}, {"cache_overhead", "int", NULL, "min=0,max=30", NULL, 0}, @@ -122,6 +133,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5}, + {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, + {"tiered_storage", "category", NULL, NULL, + confchk_WT_CONNECTION_reconfigure_tiered_storage_subconfigs, 3}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -302,6 +316,9 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_drop[] = { {"lock_wait", "boolean", NULL, NULL, NULL, 0}, {"remove_files", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_WT_SESSION_flush_tier[] = { + {"force", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_join[] = { {"bloom_bit_count", "int", NULL, "min=2,max=1000", NULL, 0}, {"bloom_false_positives", "boolean", NULL, NULL, NULL, 0}, @@ -600,6 +617,13 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_statistics_log_subconfigs[] {"timestamp", "string", NULL, NULL, NULL, 0}, {"wait", "int", NULL, "min=0,max=100000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_tiered_storage_subconfigs[] = { + {"auth_token", "string", NULL, NULL, NULL, 0}, {"enabled", "boolean", NULL, NULL, NULL, 0}, + {"local_retention", "int", NULL, "min=0,max=10000", NULL, 0}, + {"name", "string", NULL, NULL, NULL, 0}, + {"object_target_size", "int", NULL, "min=100K,max=10TB", NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_wiredtiger_open_transaction_sync_subconfigs[] = { {"enabled", "boolean", NULL, NULL, NULL, 0}, {"method", "string", NULL, "choices=[\"dsync\",\"fsync\",\"none\"]", NULL, 0}, @@ -654,6 +678,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { "\"clear\",\"tree_walk\"]", NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, + {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 5}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -730,6 +756,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { "\"clear\",\"tree_walk\"]", NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, + {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 5}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -803,6 +831,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { "\"clear\",\"tree_walk\"]", NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, + {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 5}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -874,6 +904,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { "\"clear\",\"tree_walk\"]", NULL, 0}, {"statistics_log", "category", NULL, NULL, confchk_wiredtiger_open_statistics_log_subconfigs, 6}, + {"tiered_manager", "category", NULL, NULL, confchk_tiered_manager_subconfigs, 3}, + {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 5}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," @@ -934,8 +966,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(json=false,on_close=false," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,verbose=[]", - confchk_WT_CONNECTION_reconfigure, 27}, + "tiered_manager=(threads_max=8,threads_min=1,wait=0)," + "tiered_storage=(auth_token=,local_retention=300," + "object_target_size=10M),timing_stress_for_test=,verbose=[]", + confchk_WT_CONNECTION_reconfigure, 29}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", "commit_timestamp=,durable_timestamp=,force=false," @@ -991,6 +1025,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint_wait=true,force=false,lock_wait=true," "remove_files=true", confchk_WT_SESSION_drop, 4}, + {"WT_SESSION.flush_tier", "force=false", confchk_WT_SESSION_flush_tier, 1}, {"WT_SESSION.join", "bloom_bit_count=16,bloom_false_positives=false," "bloom_hash_count=8,compare=\"eq\",count=,operation=\"and\"," @@ -1138,10 +1173,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" - ",method=fsync),use_environment=true,use_environment_priv=false," - "verbose=[],verify_metadata=false,write_through=", - confchk_wiredtiger_open, 55}, + ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," + "tiered_storage=(auth_token=,enabled=false,local_retention=300," + "name=,object_target_size=10M),timing_stress_for_test=," + "transaction_sync=(enabled=false,method=fsync)," + "use_environment=true,use_environment_priv=false,verbose=[]," + "verify_metadata=false,write_through=", + confchk_wiredtiger_open, 57}, {"wiredtiger_open_all", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," @@ -1170,11 +1208,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" - ",method=fsync),use_environment=true,use_environment_priv=false," - "verbose=[],verify_metadata=false,version=(major=0,minor=0)," - "write_through=", - confchk_wiredtiger_open_all, 56}, + ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," + "tiered_storage=(auth_token=,enabled=false,local_retention=300," + "name=,object_target_size=10M),timing_stress_for_test=," + "transaction_sync=(enabled=false,method=fsync)," + "use_environment=true,use_environment_priv=false,verbose=[]," + "verify_metadata=false,version=(major=0,minor=0),write_through=", + confchk_wiredtiger_open_all, 58}, {"wiredtiger_open_basecfg", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," @@ -1202,10 +1242,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=[],verify_metadata=false,version=(major=0," - "minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 50}, + "tiered_manager=(threads_max=8,threads_min=1,wait=0)," + "tiered_storage=(auth_token=,enabled=false,local_retention=300," + "name=,object_target_size=10M),timing_stress_for_test=," + "transaction_sync=(enabled=false,method=fsync),verbose=[]," + "verify_metadata=false,version=(major=0,minor=0),write_through=", + confchk_wiredtiger_open_basecfg, 52}, {"wiredtiger_open_usercfg", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," @@ -1233,9 +1275,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=[],verify_metadata=false,write_through=", - confchk_wiredtiger_open_usercfg, 49}, + "tiered_manager=(threads_max=8,threads_min=1,wait=0)," + "tiered_storage=(auth_token=,enabled=false,local_retention=300," + "name=,object_target_size=10M),timing_stress_for_test=," + "transaction_sync=(enabled=false,method=fsync),verbose=[]," + "verify_metadata=false,write_through=", + confchk_wiredtiger_open_usercfg, 51}, {NULL, NULL, NULL, 0}}; int diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 0c93433a59e..a2722282a24 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -93,6 +93,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) WT_TRET(__wt_capacity_server_destroy(session)); WT_TRET(__wt_checkpoint_server_destroy(session)); WT_TRET(__wt_statlog_destroy(session, true)); + WT_TRET(__wt_tiered_storage_destroy(session)); WT_TRET(__wt_sweep_destroy(session)); /* The eviction server is shut down last. */ @@ -204,6 +205,7 @@ __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) * can know if statistics are enabled or not. */ WT_RET(__wt_statlog_create(session, cfg)); + WT_RET(__wt_tiered_storage_create(session, cfg)); WT_RET(__wt_logmgr_create(session)); /* diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c index fe8c3953014..84c590944d2 100644 --- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c +++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c @@ -426,6 +426,7 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_ERR(__wt_logmgr_reconfig(session, cfg)); WT_ERR(__wt_lsm_manager_reconfig(session, cfg)); WT_ERR(__wt_statlog_create(session, cfg)); + WT_ERR(__wt_tiered_storage_create(session, cfg)); WT_ERR(__wt_sweep_config(session, cfg)); WT_ERR(__wt_timing_stress_config(session, cfg)); WT_ERR(__wt_verbose_config(session, cfg)); diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c new file mode 100644 index 00000000000..df339526bbf --- /dev/null +++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c @@ -0,0 +1,334 @@ +/*- + * Copyright (c) 2014-2020 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#ifdef __GNUC__ +#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 1) +/* + * !!! + * GCC with -Wformat-nonliteral complains about calls to strftime in this file. + * There's nothing wrong, this makes the warning go away. + */ +#pragma GCC diagnostic ignored "-Wformat-nonliteral" +#endif +#endif + +/* + * __flush_tier_once -- + * Perform one iteration of tiered storage maintenance. + */ +static void +__flush_tier_once(WT_SESSION_IMPL *session, bool force) +{ + WT_UNUSED(session); + WT_UNUSED(force); + /* + * - See if there is any "merging" work to do to prepare and create an object that is + * suitable for placing onto tiered storage. + * - Do the work to create said objects. + * - Move the objects. + */ + return; +} + +/* + * __tier_storage_remove_local -- + * Perform one iteration of tiered storage local tier removal. + */ +static int +__tier_storage_remove_local(WT_SESSION_IMPL *session, const char *uri, bool force) +{ + WT_CONFIG_ITEM cval; + WT_DECL_RET; + size_t len; + uint64_t now; + char *config, *newfile; + const char *cfg[2], *filename; + + if (uri == NULL) + return (0); + __wt_verbose(session, WT_VERB_TIERED, "Removing tree %s", uri); + filename = uri; + WT_PREFIX_SKIP_REQUIRED(session, filename, "tiered:"); + len = strlen("file:") + strlen(filename) + 1; + WT_ERR(__wt_calloc_def(session, len, &newfile)); + WT_ERR(__wt_snprintf(newfile, len, "file:%s", filename)); + + /* + * If the file:URI of the tiered object does not exist, there is nothing to do. + */ + ret = __wt_metadata_search(session, newfile, &config); + if (ret == WT_NOTFOUND) { + ret = 0; + goto err; + } + WT_ERR(ret); + + /* + * We have a local version of this tiered data. Check its metadata for when it expires and + * remove if necessary. + */ + cfg[0] = config; + cfg[1] = NULL; + WT_ERR(__wt_config_gets(session, cfg, "local_retain", &cval)); + __wt_seconds(session, &now); + if (force || (uint64_t)cval.val + S2C(session)->tiered_retain_secs >= now) + /* + * We want to remove the entry and the file. Probably do a schema_drop on the file:uri. + */ + __wt_verbose(session, WT_VERB_TIERED, "Would remove %s. Local retention expired", newfile); + +err: + __wt_free(session, config); + __wt_free(session, newfile); + return (ret); +} + +/* + * __tier_storage_remove -- + * Perform one iteration of tiered storage local tier removal. + */ +static int +__tier_storage_remove(WT_SESSION_IMPL *session, bool force) +{ + WT_UNUSED(session); + WT_UNUSED(force); + + /* + * We want to walk the metadata perhaps and for each tiered URI, call remove on its file:URI + * version. + */ + WT_RET(__tier_storage_remove_local(session, NULL, force)); + return (0); +} + +/* + * __wt_flush_tier -- + * Entry function for flush_tier method. + */ +int +__wt_flush_tier(WT_SESSION_IMPL *session, const char *config) +{ + WT_CONFIG_ITEM cval; + const char *cfg[3]; + bool force; + + WT_STAT_CONN_INCR(session, flush_tier); + if (FLD_ISSET(S2C(session)->server_flags, WT_CONN_SERVER_TIERED)) + WT_RET_MSG( + session, EINVAL, "Cannot call flush_tier when storage manager thread is configured"); + + cfg[0] = WT_CONFIG_BASE(session, WT_SESSION_flush_tier); + cfg[1] = (char *)config; + cfg[2] = NULL; + WT_RET(__wt_config_gets(session, cfg, "force", &cval)); + force = cval.val != 0; + + __flush_tier_once(session, force); + return (0); +} + +/* + * __tiered_manager_config -- + * Parse and setup the storage manager options. + */ +static int +__tiered_manager_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + WT_TIERED_MANAGER *mgr; + + conn = S2C(session); + mgr = &conn->tiered_manager; + + /* Only start the server if wait time is non-zero */ + WT_RET(__wt_config_gets(session, cfg, "tiered_manager.wait", &cval)); + mgr->wait_usecs = (uint64_t)cval.val * WT_MILLION; + if (runp != NULL) + *runp = mgr->wait_usecs != 0; + + WT_RET(__wt_config_gets(session, cfg, "tiered_manager.threads_max", &cval)); + if (cval.val > WT_TIERED_MAX_WORKERS) + WT_RET_MSG(session, EINVAL, "Maximum storage workers of %" PRIu32 " larger than %d", + (uint32_t)cval.val, WT_TIERED_MAX_WORKERS); + mgr->workers_max = (uint32_t)cval.val; + + WT_RET(__wt_config_gets(session, cfg, "tiered_manager.threads_min", &cval)); + if (cval.val < WT_TIERED_MIN_WORKERS) + WT_RET_MSG(session, EINVAL, "Minimum storage workers of %" PRIu32 " less than %d", + (uint32_t)cval.val, WT_TIERED_MIN_WORKERS); + mgr->workers_min = (uint32_t)cval.val; + WT_ASSERT(session, mgr->workers_min <= mgr->workers_max); + return (0); +} + +/* + * __tiered_config -- + * Parse and setup the storage server options. + */ +static int +__tiered_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + bool enabled; + + conn = S2C(session); + + WT_RET(__wt_config_gets(session, cfg, "tiered_storage.enabled", &cval)); + enabled = cval.val != 0; + + if (enabled) + FLD_SET(conn->tiered_flags, WT_CONN_TIERED_ENABLED); + else + FLD_CLR(conn->tiered_flags, WT_CONN_TIERED_ENABLED); + + WT_RET(__wt_config_gets(session, cfg, "tiered_storage.auth_token", &cval)); + conn->tiered_auth_token = cval.str; + + WT_RET(__wt_config_gets(session, cfg, "tiered_storage.local_retention", &cval)); + conn->tiered_retain_secs = (uint64_t)cval.val; + WT_STAT_CONN_SET(session, tiered_retention, conn->tiered_retain_secs); + + WT_RET(__wt_config_gets(session, cfg, "tiered_storage.object_target_size", &cval)); + conn->tiered_object_size = (uint64_t)cval.val; + return (__tiered_manager_config(session, cfg, runp)); +} + +/* + * __tiered_server_run_chk -- + * Check to decide if the tiered storage server should continue running. + */ +static bool +__tiered_server_run_chk(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + return ((FLD_ISSET(conn->server_flags, WT_CONN_SERVER_TIERED)) && + !F_ISSET(&conn->tiered_manager, WT_TIERED_MANAGER_SHUTDOWN)); +} + +/* + * __tiered_server -- + * The tiered storage server thread. + */ +static WT_THREAD_RET +__tiered_server(void *arg) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_ITEM path, tmp; + WT_SESSION_IMPL *session; + WT_TIERED_MANAGER *mgr; + + session = arg; + conn = S2C(session); + mgr = &conn->tiered_manager; + + WT_CLEAR(path); + WT_CLEAR(tmp); + + for (;;) { + /* Wait until the next event. */ + __wt_cond_wait(session, conn->tiered_cond, mgr->wait_usecs, __tiered_server_run_chk); + + /* Check if we're quitting or being reconfigured. */ + if (!__tiered_server_run_chk(session)) + break; + + /* + * Here is where we do work. Work we expect to do: + * + */ + __flush_tier_once(session, false); + WT_ERR(__tier_storage_remove(session, false)); + } + + if (0) { +err: + WT_IGNORE_RET(__wt_panic(session, ret, "storage server error")); + } + __wt_buf_free(session, &path); + __wt_buf_free(session, &tmp); + return (WT_THREAD_RET_VALUE); +} + +/* + * __wt_tiered_storage_create -- + * Start the tiered storage server thread. + */ +int +__wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + bool start; + + conn = S2C(session); + + /* Destroy any existing thread since we could be a reconfigure. */ + WT_RET(__wt_tiered_storage_destroy(session)); + WT_RET(__tiered_config(session, cfg, &start)); + if (!F_ISSET(conn, WT_CONN_TIERED_ENABLED) || !start) + return (0); + + /* Set first, the thread might run before we finish up. */ + FLD_SET(conn->server_flags, WT_CONN_SERVER_TIERED); + + WT_ERR(__wt_open_internal_session(conn, "storage-server", true, 0, &conn->tiered_session)); + session = conn->tiered_session; + + WT_ERR(__wt_cond_alloc(session, "storage server", &conn->tiered_cond)); + + /* Start the thread. */ + WT_ERR(__wt_thread_create(session, &conn->tiered_tid, __tiered_server, session)); + conn->tiered_tid_set = true; + + if (0) { +err: + WT_TRET(__wt_tiered_storage_destroy(session)); + } + return (ret); +} + +/* + * __wt_tiered_storage_destroy -- + * Destroy the tiered storage server thread. + */ +int +__wt_tiered_storage_destroy(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + + conn = S2C(session); + /* + * This may look a lot more like __wt_lsm_manager_destroy instead. It depends on what the final + * API looks like. For now handle it like a single internal worker thread. + */ + + /* Stop the server thread. */ + FLD_CLR(conn->server_flags, WT_CONN_SERVER_TIERED); + if (conn->tiered_tid_set) { + __wt_cond_signal(session, conn->tiered_cond); + WT_TRET(__wt_thread_join(session, &conn->tiered_tid)); + conn->tiered_tid_set = false; + } + __wt_cond_destroy(session, &conn->tiered_cond); + + /* Close the server thread's session. */ + if (conn->tiered_session != NULL) { + WT_TRET(__wt_session_close_internal(conn->tiered_session)); + conn->tiered_session = NULL; + } + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 5aa47215ace..e2a2d691a9e 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -3,6 +3,7 @@ ACM APIs ActiveState Adler's +ASAN Atomicity BLOBs BLRrVv @@ -18,7 +19,10 @@ Christoph Collet's Coverity Coverity's +CPPSuite +CSuite CURSTD +Cyclomatic DB's DBTs DHANDLE @@ -83,6 +87,7 @@ Mewhort MongoDB MongoDB's Multithreaded +MSAN NOTFOUND NSEC NUMA @@ -106,8 +111,10 @@ Seward's SiH TSC TXT +UBSAN UNC URIs +Valgrind WILLNEED WiredTiger WiredTiger's @@ -189,14 +196,17 @@ copydoc cov cp cpp +cppsuite cpu crashless crc +csuite curfile cursortype curtable customerABC cv +cyclomatic dN dNLen dNOff @@ -294,6 +304,7 @@ fuzzutil fvisibility fxray gcc +gcov gdbm ge getKey diff --git a/src/third_party/wiredtiger/src/docs/testing.dox b/src/third_party/wiredtiger/src/docs/testing.dox index 7d454d54212..b33ad279562 100644 --- a/src/third_party/wiredtiger/src/docs/testing.dox +++ b/src/third_party/wiredtiger/src/docs/testing.dox @@ -5,19 +5,21 @@ WiredTiger uses a combination of several different tools and test programs for testing WiredTiger. WiredTiger testing is done automatically using the <a -href="http://jenkins-ci.org/">Jenkins</a> continuous integration testing -framework. This allows us to be confident we don't introduce problems -in a non-standard configuration, or performance regressions. +href="http://jenkins-ci.org/">Jenkins</a> and <a +href="https://github.com/evergreen-ci/evergreen/wiki">Evergreen</a> continuous integration testing +frameworks. This allows us to be confident we don't introduce problems in a non-standard +configuration, or performance regressions. @section test_suite Unit Test Suite +@subsection test_python Python WiredTiger's primary functionality and regression testing is done in a Python unit test suite (found in the source tree under \c test/suite). The WiredTiger Python test suite includes approximately 10,000 separate -tests which are run on all platforms supported by WiredTiger. Each test +tests which are run on all platforms supported by WiredTiger. Each test is intended to test a single operation in a reproducible way, making it -easy to diagnose errors. The test suite runs multiple test cases in +easy to diagnose errors. The test suite runs multiple test cases in parallel, allowing it to be run in a relatively short period of time. The WiredTiger unit test suite includes tests that cover: @@ -32,6 +34,14 @@ least Python version 2.6). The WiredTiger test suite automatically runs as part of every commit into the WiredTiger GitHub source tree. +@subsection test_csuite CSuite +@subsection test_cppsuite CPPSuite +@subsection test_format Format +@subsection test_perf WT Perf +@subsection test_workgen Workgen + +@section testing_compatibility Compatibility Testing +@section testing_correctness Correctness Testing @section testing_performance Performance Testing Performance testing is primarily done using the \c bench/wtperf utility. @@ -42,21 +52,26 @@ WiredTiger performance tests are automatically run as part of each commit into the \c develop branch of the WiredTiger GitHub source tree and compared against previous runs to detect performance regressions. +Performance testing is also done using the \c bench/workgen utility. + @section testing_stress Stress Testing -Stress testing is primarily done using the \c test/format utility. This +Stress testing is primarily done using the \c test/format utility. This test program randomly configures a database and then runs some number of randomly selected operations, using some number of randomly selected -threads, on that database. WiredTiger stress testing is run continuously +threads, on that database. WiredTiger stress testing is run continuously on the WiredTiger GitHub \c develop branch. @section testing_threads Concurrency testing Concurrency testing is primarily done using the \c test/format utility. Additionally, the \c test/thread and \c test/fops test utilities test -specific heavily threaded operations. WiredTiger concurrency testing +specific heavily threaded operations. WiredTiger concurrency testing is run continuously on the WiredTiger GitHub \c develop branch. +@section testing_runtime Runtime correctness checking +ASAN, MSAN, UBSAN, Valgrind + @section testing_static Static analysis WiredTiger static analysis is done using three tools: @@ -74,4 +89,43 @@ of the UNIX lint tool. - The University of Illinois LLVM project's <a href="http://clang-analyzer.llvm.org/">Clang Static Analyzer</a>. +@section testing_coverage Coverage + +@subsection coverage_tests Tests +The <a href="https://github.com/wiredtiger/wiredtiger/blob/develop/test/test_coverage.md"> test +coverage documentation</a> is generated in each branch, the link provided is to the current +\c develop version of the document. + +@subsection coverage_tags Tagging scheme +Each test file shall contain a tag using the following scheme: + +@code{.console} +[TEST_TAGS] +<COMPONENT>:<TESTING_TYPE>:<TESTING_AREA> +[END_TAGS] +@endcode + +One test file can have multiple tags: + +@code{.console} +[TEST_TAGS] +backup:correctness:full_backup +checkpoints:correctness:checkpoint_data +caching_eviction:correctness:written_data +[END_TAGS] +`@endcode + +If a test file shall be ignored, the following tag can be used: + +@code{.console} +[TEST_TAGS] +ignored_file +[END_TAGS] +@endcode + +@subsection coverage_code Code +WiredTiger code coverage is measured by gcov. + +@subsection coverage_cyclomatic Cyclomatic Complexity + */ diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 438e888a6b6..58fd2bd2ac6 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -228,6 +228,9 @@ struct __wt_ovfl_reuse { * point. If we need to store more information, we can potentially tack extra information at the end * of the "value" buffer and then use bit flags within the update type to determine how to interpret * it. + * + * We also configure a larger than default internal page size to accommodate for larger history + * store keys. We do that to reduce the chances of having to create overflow keys on the page. */ #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY #define WT_HS_COMPRESSOR "snappy" @@ -239,6 +242,7 @@ struct __wt_ovfl_reuse { #define WT_HS_CONFIG \ "key_format=" WT_HS_KEY_FORMAT ",value_format=" WT_HS_VALUE_FORMAT \ ",block_compressor=" WT_HS_COMPRESSOR \ + ",internal_page_max=16KB" \ ",leaf_value_max=64MB" \ ",prefix_compression=false" diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 8b405657f28..8b87654bfb3 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -75,35 +75,36 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_SESSION_compact 21 #define WT_CONFIG_ENTRY_WT_SESSION_create 22 #define WT_CONFIG_ENTRY_WT_SESSION_drop 23 -#define WT_CONFIG_ENTRY_WT_SESSION_join 24 -#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 25 -#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 26 -#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 27 -#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 28 -#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 29 -#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 30 -#define WT_CONFIG_ENTRY_WT_SESSION_rename 31 -#define WT_CONFIG_ENTRY_WT_SESSION_reset 32 -#define WT_CONFIG_ENTRY_WT_SESSION_reset_snapshot 33 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 34 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 35 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 36 -#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 37 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 38 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 39 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 40 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 41 -#define WT_CONFIG_ENTRY_colgroup_meta 42 -#define WT_CONFIG_ENTRY_file_config 43 -#define WT_CONFIG_ENTRY_file_meta 44 -#define WT_CONFIG_ENTRY_index_meta 45 -#define WT_CONFIG_ENTRY_lsm_meta 46 -#define WT_CONFIG_ENTRY_table_meta 47 -#define WT_CONFIG_ENTRY_tiered_meta 48 -#define WT_CONFIG_ENTRY_wiredtiger_open 49 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 50 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 51 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 52 +#define WT_CONFIG_ENTRY_WT_SESSION_flush_tier 24 +#define WT_CONFIG_ENTRY_WT_SESSION_join 25 +#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 26 +#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 27 +#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 28 +#define WT_CONFIG_ENTRY_WT_SESSION_prepare_transaction 29 +#define WT_CONFIG_ENTRY_WT_SESSION_query_timestamp 30 +#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 31 +#define WT_CONFIG_ENTRY_WT_SESSION_rename 32 +#define WT_CONFIG_ENTRY_WT_SESSION_reset 33 +#define WT_CONFIG_ENTRY_WT_SESSION_reset_snapshot 34 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 35 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 36 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 37 +#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 38 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 39 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 40 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 41 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 42 +#define WT_CONFIG_ENTRY_colgroup_meta 43 +#define WT_CONFIG_ENTRY_file_config 44 +#define WT_CONFIG_ENTRY_file_meta 45 +#define WT_CONFIG_ENTRY_index_meta 46 +#define WT_CONFIG_ENTRY_lsm_meta 47 +#define WT_CONFIG_ENTRY_table_meta 48 +#define WT_CONFIG_ENTRY_tiered_meta 49 +#define WT_CONFIG_ENTRY_wiredtiger_open 50 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 51 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 52 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 53 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 9eed3f4f2ba..1d89514f143 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -365,6 +365,25 @@ struct __wt_connection_impl { const char *stat_stamp; /* Statistics log entry timestamp */ uint64_t stat_usecs; /* Statistics log period */ + WT_SESSION_IMPL *tiered_session; /* Tiered thread session */ + wt_thread_t tiered_tid; /* Tiered thread */ + bool tiered_tid_set; /* Tiered thread set */ + WT_CONDVAR *tiered_cond; /* Tiered wait mutex */ + uint64_t tiered_object_size; /* Tiered object size */ + uint64_t tiered_retain_secs; /* Tiered period */ + const char *tiered_auth_token; /* Tiered authentication cookie */ + + WT_TIERED_MANAGER tiered_manager; /* Tiered worker thread information */ + bool tiered_server_running; /* Internal tiered server operating */ + + uint32_t tiered_threads_max; /* Max tiered threads */ + uint32_t tiered_threads_min; /* Min tiered threads */ + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_CONN_TIERED_ENABLED 0x1u /* Shared tiered is configured */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t tiered_flags; /* Global tiered configuration */ + /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_CONN_LOG_ARCHIVE 0x001u /* Archive is enabled */ #define WT_CONN_LOG_CONFIG_ENABLED 0x002u /* Logging is configured */ @@ -558,6 +577,7 @@ struct __wt_connection_impl { #define WT_CONN_SERVER_LSM 0x08u #define WT_CONN_SERVER_STATISTICS 0x10u #define WT_CONN_SERVER_SWEEP 0x20u +#define WT_CONN_SERVER_TIERED 0x40u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t server_flags; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 9941c9ba34a..c7490d643b6 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -725,6 +725,8 @@ extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path extern int __wt_filename_construct(WT_SESSION_IMPL *session, const char *path, const char *file_prefix, uintmax_t id_1, uint32_t id_2, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_flush_tier(WT_SESSION_IMPL *session, const char *config) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, uint32_t open_flags, uint32_t flags, WT_FSTREAM **fstrp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_fsync_background(WT_SESSION_IMPL *session) @@ -1453,6 +1455,10 @@ extern int __wt_tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_storage_create(WT_SESSION_IMPL *session, const char *cfg[]) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tiered_storage_destroy(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_truncate(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_worker(WT_SESSION_IMPL *session, const char *uri, diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index ca7f85f33df..99bcb538f59 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -578,6 +578,8 @@ struct __wt_connection_stats { int64_t perf_hist_opwrite_latency_lt1000; int64_t perf_hist_opwrite_latency_lt10000; int64_t perf_hist_opwrite_latency_gt10000; + int64_t rec_overflow_key_internal; + int64_t rec_overflow_key_leaf; int64_t rec_maximum_seconds; int64_t rec_pages_with_prepare; int64_t rec_pages_with_ts; @@ -779,6 +781,8 @@ struct __wt_connection_stats { int64_t rec_time_window_durable_stop_ts; int64_t rec_time_window_stop_ts; int64_t rec_time_window_stop_txn; + int64_t flush_tier; + int64_t tiered_retention; int64_t txn_read_race_prepare_update; int64_t txn_rts_hs_stop_older_than_newer_start; int64_t txn_rts_keys_removed; @@ -994,6 +998,8 @@ struct __wt_dsrc_stats { int64_t rec_time_window_durable_stop_ts; int64_t rec_time_window_stop_ts; int64_t rec_time_window_stop_txn; + int64_t flush_tier; + int64_t tiered_retention; int64_t txn_read_race_prepare_update; int64_t txn_rts_hs_stop_older_than_newer_start; int64_t txn_rts_keys_removed; diff --git a/src/third_party/wiredtiger/src/include/tiered.h b/src/third_party/wiredtiger/src/include/tiered.h index ac4343661c5..caeb2226bc4 100644 --- a/src/third_party/wiredtiger/src/include/tiered.h +++ b/src/third_party/wiredtiger/src/include/tiered.h @@ -7,6 +7,26 @@ */ /* + * WT_TIERED_MANAGER -- + * A structure that holds resources used to manage any tiered storage + * for the whole database. + */ +struct __wt_tiered_manager { + uint64_t wait_usecs; /* Wait time period */ + uint32_t workers; /* Current number of workers */ + uint32_t workers_max; + uint32_t workers_min; + +#define WT_TIERED_MAX_WORKERS 20 +#define WT_TIERED_MIN_WORKERS 1 + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_TIERED_MANAGER_SHUTDOWN 0x1u /* Manager has shut down */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t flags; +}; + +/* * WT_CURSOR_TIERED -- * An tiered cursor. */ diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index bcfd32f08c6..efcde12d1db 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -791,6 +791,19 @@ struct __wt_session { */ int __F(reconfigure)(WT_SESSION *session, const char *config); +#ifndef DOXYGEN + /*! + * Initiate a single operation to manage tiered storage. + * + * @param session the session handle + * @configstart{WT_SESSION.flush_tier, see dist/api_data.py} + * @config{force, force sharing of all data., a boolean flag; default \c false.} + * @configend + * @errors + */ + int __F(flush_tier)(WT_SESSION *session, const char *config); +#endif + /*! * Return information about an error as a string. * @@ -2197,6 +2210,18 @@ struct __wt_connection { * records; setting this value above 0 configures statistics logging., an integer between 0 * and 100000; default \c 0.} * @config{ ),,} + * @config{tiered_storage = (, enable tiered storage. Enabling tiered storage may use one + * session from the configured session_max., a set of related configuration options defined + * below.} + * @config{ auth_token, authentication token string., a + * string; default empty.} + * @config{ local_retention, time in seconds + * to retain data on tiered storage on the local tier for faster read access., an integer + * between 0 and 10000; default \c 300.} + * @config{ object_target_size, + * the approximate size of objects before creating them on the tiered storage tier., an + * integer between 100K and 10TB; default \c 10M.} + * @config{ ),,} * @config{verbose, enable messages for various events. Options are given as a list\, such * as <code>"verbose=[evictserver\,read]"</code>., a list\, with values chosen from the * following options: \c "api"\, \c "backup"\, \c "block"\, \c "checkpoint"\, \c @@ -5342,554 +5367,562 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1263 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ #define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1264 +/*! reconciliation: internal-page overflow keys */ +#define WT_STAT_CONN_REC_OVERFLOW_KEY_INTERNAL 1265 +/*! reconciliation: leaf-page overflow keys */ +#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1266 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1265 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1267 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1266 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1268 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1267 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1269 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1268 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1270 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1269 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1271 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1270 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1272 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1271 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1273 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1272 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1274 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1273 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1275 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1274 +#define WT_STAT_CONN_SESSION_OPEN 1276 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1275 +#define WT_STAT_CONN_SESSION_QUERY_TS 1277 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1276 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1278 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1277 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1279 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1278 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1280 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1279 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1281 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1280 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1282 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1281 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1283 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1282 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1284 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1283 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1285 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1284 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1286 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1285 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1287 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1286 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1288 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1287 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1289 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1288 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1290 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1289 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1291 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1290 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1292 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1291 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1293 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1292 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1294 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1293 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1295 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1294 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1296 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1295 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1297 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1296 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1298 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1297 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1299 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1298 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1300 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1299 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1301 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1300 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1302 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1301 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1303 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1302 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1304 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1303 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1305 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1304 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1306 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1305 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1307 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1306 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1308 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1307 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1309 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1308 +#define WT_STAT_CONN_PAGE_SLEEP 1310 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1309 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1311 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1310 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1312 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1311 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1313 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1312 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1314 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1313 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1315 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1314 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1316 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1315 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1317 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1316 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1318 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1317 +#define WT_STAT_CONN_TXN_PREPARE 1319 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1318 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1320 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1319 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1321 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1320 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1322 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1321 +#define WT_STAT_CONN_TXN_QUERY_TS 1323 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1322 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1324 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1323 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1325 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1324 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1326 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1325 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1327 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1326 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1328 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1327 +#define WT_STAT_CONN_TXN_RTS 1329 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1328 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1330 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1329 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1331 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1330 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1332 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1331 +#define WT_STAT_CONN_TXN_SET_TS 1333 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1332 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1334 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1333 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1335 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1334 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1336 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1335 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1337 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1336 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1338 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1337 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1339 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1338 +#define WT_STAT_CONN_TXN_BEGIN 1340 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1339 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1341 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1340 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1342 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1341 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1343 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1342 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1344 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1343 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1345 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1344 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1346 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1345 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1347 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1346 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1348 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1347 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1349 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1348 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1350 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1349 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1351 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1350 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1352 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1351 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1353 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1352 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1354 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1353 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1355 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1354 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1356 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1355 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1357 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1356 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1358 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1357 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1359 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1358 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1360 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1359 +#define WT_STAT_CONN_TXN_CHECKPOINT 1361 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1360 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1362 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1361 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1363 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1362 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1364 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1363 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1365 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1364 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1366 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1365 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1367 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1366 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1368 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1367 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1369 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1368 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1370 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1369 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1371 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1370 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1372 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1371 +#define WT_STAT_CONN_TXN_SYNC 1373 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1372 +#define WT_STAT_CONN_TXN_COMMIT 1374 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1373 +#define WT_STAT_CONN_TXN_ROLLBACK 1375 /*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1374 +#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1376 /*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1375 +#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1377 /*! cache: bytes currently in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INUSE 1376 +#define WT_STAT_CONN_CACHE_BYTES_INUSE 1378 /*! cache: bytes dirty in the cache cumulative */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY_TOTAL 1377 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY_TOTAL 1379 /*! cache: bytes read into cache */ -#define WT_STAT_CONN_CACHE_BYTES_READ 1378 +#define WT_STAT_CONN_CACHE_BYTES_READ 1380 /*! cache: bytes written from cache */ -#define WT_STAT_CONN_CACHE_BYTES_WRITE 1379 +#define WT_STAT_CONN_CACHE_BYTES_WRITE 1381 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1380 +#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1382 /*! cache: eviction walk target pages histogram - 0-9 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1381 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT10 1383 /*! cache: eviction walk target pages histogram - 10-31 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1382 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT32 1384 /*! cache: eviction walk target pages histogram - 128 and higher */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1383 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_GE128 1385 /*! cache: eviction walk target pages histogram - 32-63 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1384 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT64 1386 /*! cache: eviction walk target pages histogram - 64-128 */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1385 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_LT128 1387 /*! * cache: eviction walk target pages reduced due to history store cache * pressure */ -#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_REDUCED 1386 +#define WT_STAT_CONN_CACHE_EVICTION_TARGET_PAGE_REDUCED 1388 /*! cache: eviction walks abandoned */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1387 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1389 /*! cache: eviction walks gave up because they restarted their walk twice */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1388 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STOPPED 1390 /*! * cache: eviction walks gave up because they saw too many pages and * found no candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1389 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_NO_TARGETS 1391 /*! * cache: eviction walks gave up because they saw too many pages and * found too few candidates */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1390 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_GAVE_UP_RATIO 1392 /*! cache: eviction walks reached end of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1391 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ENDED 1393 /*! cache: eviction walks restarted */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_RESTART 1392 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_RESTART 1394 /*! cache: eviction walks started from root of tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1393 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_FROM_ROOT 1395 /*! cache: eviction walks started from saved location in tree */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1394 +#define WT_STAT_CONN_CACHE_EVICTION_WALK_SAVED_POS 1396 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1395 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1397 /*! cache: history store table insert calls */ -#define WT_STAT_CONN_CACHE_HS_INSERT 1396 +#define WT_STAT_CONN_CACHE_HS_INSERT 1398 /*! cache: history store table insert calls that returned restart */ -#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1397 +#define WT_STAT_CONN_CACHE_HS_INSERT_RESTART 1399 /*! * cache: history store table out-of-order resolved updates that lose * their durable timestamp */ -#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1398 +#define WT_STAT_CONN_CACHE_HS_ORDER_LOSE_DURABLE_TIMESTAMP 1400 /*! * cache: history store table out-of-order updates that were fixed up by * moving existing records */ -#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_MOVE 1399 +#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_MOVE 1401 /*! * cache: history store table out-of-order updates that were fixed up * during insertion */ -#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_INSERT 1400 +#define WT_STAT_CONN_CACHE_HS_ORDER_FIXUP_INSERT 1402 /*! cache: history store table reads */ -#define WT_STAT_CONN_CACHE_HS_READ 1401 +#define WT_STAT_CONN_CACHE_HS_READ 1403 /*! cache: history store table reads missed */ -#define WT_STAT_CONN_CACHE_HS_READ_MISS 1402 +#define WT_STAT_CONN_CACHE_HS_READ_MISS 1404 /*! cache: history store table reads requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1403 +#define WT_STAT_CONN_CACHE_HS_READ_SQUASH 1405 /*! * cache: history store table truncation by rollback to stable to remove * an unstable update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1404 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS_UNSTABLE 1406 /*! * cache: history store table truncation by rollback to stable to remove * an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1405 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_RTS 1407 /*! cache: history store table truncation to remove an update */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1406 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE 1408 /*! * cache: history store table truncation to remove range of updates due * to key being removed from the data page during reconciliation */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1407 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_ONPAGE_REMOVAL 1409 /*! * cache: history store table truncation to remove range of updates due * to non timestamped update on data page */ -#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_NON_TS 1408 +#define WT_STAT_CONN_CACHE_HS_KEY_TRUNCATE_NON_TS 1410 /*! cache: history store table writes requiring squashed modifies */ -#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1409 +#define WT_STAT_CONN_CACHE_HS_WRITE_SQUASH 1411 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1410 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1412 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1411 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1413 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1412 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1414 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1413 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1415 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1414 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1416 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1415 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1417 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1416 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1418 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1417 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1419 /*! cache: page written requiring history store records */ -#define WT_STAT_CONN_CACHE_WRITE_HS 1418 +#define WT_STAT_CONN_CACHE_WRITE_HS 1420 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1419 +#define WT_STAT_CONN_CACHE_READ 1421 /*! cache: pages read into cache after truncate */ -#define WT_STAT_CONN_CACHE_READ_DELETED 1420 +#define WT_STAT_CONN_CACHE_READ_DELETED 1422 /*! cache: pages read into cache after truncate in prepare state */ -#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1421 +#define WT_STAT_CONN_CACHE_READ_DELETED_PREPARED 1423 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1422 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1424 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1423 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1425 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1424 +#define WT_STAT_CONN_CACHE_WRITE 1426 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1425 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1427 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1426 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1428 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1427 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1429 /*! checkpoint-cleanup: pages added for eviction */ -#define WT_STAT_CONN_CC_PAGES_EVICT 1428 +#define WT_STAT_CONN_CC_PAGES_EVICT 1430 /*! checkpoint-cleanup: pages removed */ -#define WT_STAT_CONN_CC_PAGES_REMOVED 1429 +#define WT_STAT_CONN_CC_PAGES_REMOVED 1431 /*! checkpoint-cleanup: pages skipped during tree walk */ -#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1430 +#define WT_STAT_CONN_CC_PAGES_WALK_SKIPPED 1432 /*! checkpoint-cleanup: pages visited */ -#define WT_STAT_CONN_CC_PAGES_VISITED 1431 +#define WT_STAT_CONN_CC_PAGES_VISITED 1433 /*! cursor: Total number of entries skipped by cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1432 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_TOTAL 1434 /*! cursor: Total number of entries skipped by cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1433 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_TOTAL 1435 /*! * cursor: Total number of entries skipped to position the history store * cursor */ -#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1434 +#define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1436 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1435 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1437 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1436 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1438 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1437 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1439 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1438 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1440 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1439 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1441 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1440 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1442 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1441 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1443 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1442 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1444 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1443 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1445 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1444 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1446 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1445 +#define WT_STAT_CONN_REC_PAGES 1447 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1446 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1448 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1447 +#define WT_STAT_CONN_REC_PAGE_DELETE 1449 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1448 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1450 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1449 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1451 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1450 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1452 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1451 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1453 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1452 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1454 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1453 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1455 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1454 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1456 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1455 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1457 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1456 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1458 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1457 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1459 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1458 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1460 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1459 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1461 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1460 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1462 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1461 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1463 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1462 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1464 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1463 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1465 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1464 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1466 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1465 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1467 +/*! session: flush_tier operation calls */ +#define WT_STAT_CONN_FLUSH_TIER 1468 +/*! session: tiered storage local retention time (secs) */ +#define WT_STAT_CONN_TIERED_RETENTION 1469 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1466 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1470 /*! * transaction: rollback to stable hs records with stop timestamps older * than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1467 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1471 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1468 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1472 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1469 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1473 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1470 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1474 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1471 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1475 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1472 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1476 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1473 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1477 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1474 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1478 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1475 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1479 /*! * @} @@ -6480,29 +6513,33 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2196 /*! reconciliation: records written including a stop transaction ID */ #define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2197 +/*! session: flush_tier operation calls */ +#define WT_STAT_DSRC_FLUSH_TIER 2198 +/*! session: tiered storage local retention time (secs) */ +#define WT_STAT_DSRC_TIERED_RETENTION 2199 /*! transaction: race to read prepared update retry */ -#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2198 +#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2200 /*! * transaction: rollback to stable hs records with stop timestamps older * than newer records */ -#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2199 +#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2201 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2200 +#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2202 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2201 +#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2203 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2202 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2204 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2203 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2205 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2204 +#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2206 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2205 +#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2207 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2206 +#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2208 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2207 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2209 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 62b97d7e047..91d36e90221 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -317,6 +317,8 @@ struct __wt_thread_group; typedef struct __wt_thread_group WT_THREAD_GROUP; struct __wt_tiered; typedef struct __wt_tiered WT_TIERED; +struct __wt_tiered_manager; +typedef struct __wt_tiered_manager WT_TIERED_MANAGER; struct __wt_time_aggregate; typedef struct __wt_time_aggregate WT_TIME_AGGREGATE; struct __wt_time_window; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 3ab39e65f0a..6e408adca4b 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -77,7 +77,7 @@ __rec_cell_build_int_key( /* Create an overflow object if the data won't fit. */ if (size > btree->maxintlkey) { - WT_STAT_DATA_INCR(session, rec_overflow_key_internal); + WT_STAT_CONN_DATA_INCR(session, rec_overflow_key_internal); *is_ovflp = true; return (__wt_rec_cell_build_ovfl(session, r, key, WT_CELL_KEY_OVFL, NULL, 0)); @@ -163,7 +163,7 @@ __rec_cell_build_leaf_key( * compressed. */ if (pfx == 0) { - WT_STAT_DATA_INCR(session, rec_overflow_key_leaf); + WT_STAT_CONN_DATA_INCR(session, rec_overflow_key_leaf); *is_ovflp = true; return (__wt_rec_cell_build_ovfl(session, r, key, WT_CELL_KEY_OVFL, NULL, 0)); diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 44ea3324759..6c5d672900b 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -1932,6 +1932,23 @@ __wt_session_strerror(WT_SESSION *wt_session, int error) } /* + * __session_flush_tier -- + * Wrapper for the flush_tier method. + */ +static int +__session_flush_tier(WT_SESSION *wt_session, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + SESSION_API_CALL_NOCONF(session, flush_tier); + ret = __wt_flush_tier(session, config); +err: + API_END_RET(session, ret); +} + +/* * __wt_session_breakpoint -- * A place to put a breakpoint, if you need one, or call some check code. */ @@ -1952,25 +1969,26 @@ __open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const WT_SESSION_IMPL **sessionp) { static const WT_SESSION - stds = {NULL, NULL, __session_close, __session_reconfigure, __wt_session_strerror, - __session_open_cursor, __session_alter, __session_create, __wt_session_compact, - __session_drop, __session_join, __session_log_flush, __session_log_printf, __session_rename, - __session_reset, __session_salvage, __session_truncate, __session_upgrade, __session_verify, - __session_begin_transaction, __session_commit_transaction, __session_prepare_transaction, - __session_reset_snapshot, __session_rollback_transaction, __session_timestamp_transaction, - __session_query_timestamp, __session_checkpoint, __session_transaction_pinned_range, - __session_transaction_sync, __wt_session_breakpoint}, - stds_readonly = {NULL, NULL, __session_close, __session_reconfigure, __wt_session_strerror, - __session_open_cursor, __session_alter_readonly, __session_create_readonly, - __wt_session_compact_readonly, __session_drop_readonly, __session_join, - __session_log_flush_readonly, __session_log_printf_readonly, __session_rename_readonly, - __session_reset, __session_salvage_readonly, __session_truncate_readonly, - __session_upgrade_readonly, __session_verify, __session_begin_transaction, - __session_commit_transaction, __session_prepare_transaction_readonly, - __session_reset_snapshot, __session_rollback_transaction, __session_timestamp_transaction, - __session_query_timestamp, __session_checkpoint_readonly, - __session_transaction_pinned_range, __session_transaction_sync_readonly, - __wt_session_breakpoint}; + stds = {NULL, NULL, __session_close, __session_reconfigure, __session_flush_tier, + __wt_session_strerror, __session_open_cursor, __session_alter, __session_create, + __wt_session_compact, __session_drop, __session_join, __session_log_flush, + __session_log_printf, __session_rename, __session_reset, __session_salvage, + __session_truncate, __session_upgrade, __session_verify, __session_begin_transaction, + __session_commit_transaction, __session_prepare_transaction, __session_reset_snapshot, + __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp, + __session_checkpoint, __session_transaction_pinned_range, __session_transaction_sync, + __wt_session_breakpoint}, + stds_readonly = {NULL, NULL, __session_close, __session_reconfigure, __session_flush_tier, + __wt_session_strerror, __session_open_cursor, __session_alter_readonly, + __session_create_readonly, __wt_session_compact_readonly, __session_drop_readonly, + __session_join, __session_log_flush_readonly, __session_log_printf_readonly, + __session_rename_readonly, __session_reset, __session_salvage_readonly, + __session_truncate_readonly, __session_upgrade_readonly, __session_verify, + __session_begin_transaction, __session_commit_transaction, + __session_prepare_transaction_readonly, __session_reset_snapshot, + __session_rollback_transaction, __session_timestamp_transaction, __session_query_timestamp, + __session_checkpoint_readonly, __session_transaction_pinned_range, + __session_transaction_sync_readonly, __wt_session_breakpoint}; WT_DECL_RET; WT_SESSION_IMPL *session, *session_ret; uint32_t i; diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 87a9cc4de0e..c0c776bde2e 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -205,6 +205,8 @@ static const char *const __stats_dsrc_desc[] = { "reconciliation: records written including a stop durable timestamp", "reconciliation: records written including a stop timestamp", "reconciliation: records written including a stop transaction ID", + "session: flush_tier operation calls", + "session: tiered storage local retention time (secs)", "transaction: race to read prepared update retry", "transaction: rollback to stable hs records with stop timestamps older than newer records", "transaction: rollback to stable keys removed", @@ -453,6 +455,8 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->rec_time_window_durable_stop_ts = 0; stats->rec_time_window_stop_ts = 0; stats->rec_time_window_stop_txn = 0; + stats->flush_tier = 0; + /* not clearing tiered_retention */ stats->txn_read_race_prepare_update = 0; stats->txn_rts_hs_stop_older_than_newer_start = 0; stats->txn_rts_keys_removed = 0; @@ -688,6 +692,8 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->rec_time_window_durable_stop_ts += from->rec_time_window_durable_stop_ts; to->rec_time_window_stop_ts += from->rec_time_window_stop_ts; to->rec_time_window_stop_txn += from->rec_time_window_stop_txn; + to->flush_tier += from->flush_tier; + to->tiered_retention += from->tiered_retention; to->txn_read_race_prepare_update += from->txn_read_race_prepare_update; to->txn_rts_hs_stop_older_than_newer_start += from->txn_rts_hs_stop_older_than_newer_start; to->txn_rts_keys_removed += from->txn_rts_keys_removed; @@ -928,6 +934,8 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->rec_time_window_durable_stop_ts += WT_STAT_READ(from, rec_time_window_durable_stop_ts); to->rec_time_window_stop_ts += WT_STAT_READ(from, rec_time_window_stop_ts); to->rec_time_window_stop_txn += WT_STAT_READ(from, rec_time_window_stop_txn); + to->flush_tier += WT_STAT_READ(from, flush_tier); + to->tiered_retention += WT_STAT_READ(from, tiered_retention); to->txn_read_race_prepare_update += WT_STAT_READ(from, txn_read_race_prepare_update); to->txn_rts_hs_stop_older_than_newer_start += WT_STAT_READ(from, txn_rts_hs_stop_older_than_newer_start); @@ -1215,6 +1223,8 @@ static const char *const __stats_connection_desc[] = { "perf: operation write latency histogram (bucket 3) - 500-999us", "perf: operation write latency histogram (bucket 4) - 1000-9999us", "perf: operation write latency histogram (bucket 5) - 10000us+", + "reconciliation: internal-page overflow keys", + "reconciliation: leaf-page overflow keys", "reconciliation: maximum seconds spent in a reconciliation call", "reconciliation: page reconciliation calls that resulted in values with prepared transaction " "metadata", @@ -1420,6 +1430,8 @@ static const char *const __stats_connection_desc[] = { "reconciliation: records written including a stop durable timestamp", "reconciliation: records written including a stop timestamp", "reconciliation: records written including a stop transaction ID", + "session: flush_tier operation calls", + "session: tiered storage local retention time (secs)", "transaction: race to read prepared update retry", "transaction: rollback to stable hs records with stop timestamps older than newer records", "transaction: rollback to stable keys removed", @@ -1735,6 +1747,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->perf_hist_opwrite_latency_lt1000 = 0; stats->perf_hist_opwrite_latency_lt10000 = 0; stats->perf_hist_opwrite_latency_gt10000 = 0; + stats->rec_overflow_key_internal = 0; + stats->rec_overflow_key_leaf = 0; /* not clearing rec_maximum_seconds */ stats->rec_pages_with_prepare = 0; stats->rec_pages_with_ts = 0; @@ -1936,6 +1950,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->rec_time_window_durable_stop_ts = 0; stats->rec_time_window_stop_ts = 0; stats->rec_time_window_stop_txn = 0; + stats->flush_tier = 0; + /* not clearing tiered_retention */ stats->txn_read_race_prepare_update = 0; stats->txn_rts_hs_stop_older_than_newer_start = 0; stats->txn_rts_keys_removed = 0; @@ -2248,6 +2264,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->perf_hist_opwrite_latency_lt1000 += WT_STAT_READ(from, perf_hist_opwrite_latency_lt1000); to->perf_hist_opwrite_latency_lt10000 += WT_STAT_READ(from, perf_hist_opwrite_latency_lt10000); to->perf_hist_opwrite_latency_gt10000 += WT_STAT_READ(from, perf_hist_opwrite_latency_gt10000); + to->rec_overflow_key_internal += WT_STAT_READ(from, rec_overflow_key_internal); + to->rec_overflow_key_leaf += WT_STAT_READ(from, rec_overflow_key_leaf); to->rec_maximum_seconds += WT_STAT_READ(from, rec_maximum_seconds); to->rec_pages_with_prepare += WT_STAT_READ(from, rec_pages_with_prepare); to->rec_pages_with_ts += WT_STAT_READ(from, rec_pages_with_ts); @@ -2462,6 +2480,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->rec_time_window_durable_stop_ts += WT_STAT_READ(from, rec_time_window_durable_stop_ts); to->rec_time_window_stop_ts += WT_STAT_READ(from, rec_time_window_stop_ts); to->rec_time_window_stop_txn += WT_STAT_READ(from, rec_time_window_stop_txn); + to->flush_tier += WT_STAT_READ(from, flush_tier); + to->tiered_retention += WT_STAT_READ(from, tiered_retention); to->txn_read_race_prepare_update += WT_STAT_READ(from, txn_read_race_prepare_update); to->txn_rts_hs_stop_older_than_newer_start += WT_STAT_READ(from, txn_rts_hs_stop_older_than_newer_start); diff --git a/src/third_party/wiredtiger/test/cppsuite/test_config.c b/src/third_party/wiredtiger/test/cppsuite/test_config.c index 6d1585172c8..7c159f250e4 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_config.c +++ b/src/third_party/wiredtiger/test/cppsuite/test_config.c @@ -3,13 +3,22 @@ #include "wt_internal.h" static const WT_CONFIG_CHECK confchk_poc_test[] = { - {"collection_count", "int", NULL, "min=1,max=10", NULL, 0}, - {"key_size", "int", NULL, "min=1,max=10000", NULL, 0}, - {"values", "string", NULL, "choices=[\"first\",\"second\",\"third\"]", NULL, 0}, - {NULL, NULL, NULL, NULL, NULL, 0}}; + {"collection_count", "int", NULL, "min=0,max=200000", NULL, 0}, + {"insert_config", "string", NULL, NULL, NULL, 0}, + {"insert_threads", "int", NULL, "min=0,max=20", NULL, 0}, + {"key_count", "int", NULL, "min=0,max=1000000", NULL, 0}, + {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, + {"read_threads", "int", NULL, "min=0,max=100", NULL, 0}, + {"update_config", "string", NULL, NULL, NULL, 0}, + {"update_threads", "int", NULL, "min=0,max=20", NULL, 0}, + {"value_size", "int", NULL, "min=0,max=10000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_ENTRY config_entries[] = { - {"poc_test", "collection_count=1,key_size=10,values=first", confchk_poc_test, 3}, + {"poc_test", + "collection_count=1,insert_config=,insert_threads=0,key_count=0," + "key_size=0,read_threads=0,update_config=,update_threads=0," + "value_size=0", + confchk_poc_test, 9}, {NULL, NULL, NULL, 0}}; /* diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h new file mode 100644 index 00000000000..be6f3500d1f --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/api_const.h @@ -0,0 +1,15 @@ +/* Include guard. */ +#ifndef API_CONST_H +#define API_CONST_H + +/* Define all constants related to WiredTiger APIs and testing. */ +namespace test_harness { + +static const char *CONNECTION_CREATE = "create"; +static const char *COLLECTION_COUNT = "collection_count"; +static const char *KEY_COUNT = "key_count"; +static const char *VALUE_SIZE = "value_size"; + +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration_settings.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration_settings.h index 9f1d43773c9..a530afceb27 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration_settings.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/configuration_settings.h @@ -2,16 +2,13 @@ #ifndef CONFIGURATION_SETTINGS_H #define CONFIGURATION_SETTINGS_H -#include "wt_internal.h" #include <string> #include <stdexcept> +#include "wt_internal.h" + namespace test_harness { class configuration { - private: - std::string _config; - WT_CONFIG_PARSER *_config_parser; - public: configuration(const char *test_config_name, const char *config) : _config(config) { @@ -45,7 +42,7 @@ class configuration { std::string get_config() { - return _config; + return (_config); } /* @@ -93,14 +90,18 @@ class configuration { int next(WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) { - return _config_parser->next(_config_parser, key, value); + return (_config_parser->next(_config_parser, key, value)); } int get(const char *key, WT_CONFIG_ITEM *value) { - return _config_parser->get(_config_parser, key, value); + return (_config_parser->get(_config_parser, key, value)); } + + private: + std::string _config; + WT_CONFIG_PARSER *_config_parser; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/random_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/random_generator.h new file mode 100644 index 00000000000..9e7d7021803 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/random_generator.h @@ -0,0 +1,54 @@ +/* Include guard. */ +#ifndef RANDOM_GENERATOR_H +#define RANDOM_GENERATOR_H + +#include <algorithm> +#include <cstddef> +#include <random> +#include <string> + +namespace test_harness { + +/* Helper class to generate random values. */ +class random_generator { + public: + static random_generator * + get_instance() + { + if (!_instance) + _instance = new random_generator; + return (_instance); + } + + std::string + generate_string(std::size_t length) + { + std::string random_string; + + if (length == 0) + throw std::invalid_argument("random_generator.generate_string: 0 is an invalid length"); + + for (std::size_t i = 0; i < length; ++i) + random_string += _characters[_distribution(_generator)]; + + return (random_string); + } + + private: + random_generator() + { + _generator = std::mt19937(_random_device()); + _distribution = std::uniform_int_distribution<>(0, _characters.size() - 1); + } + + static random_generator *_instance; + std::mt19937 _generator; + std::random_device _random_device; + std::uniform_int_distribution<> _distribution; + const std::string _characters = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; +}; +random_generator *random_generator::_instance = 0; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/test_harness.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/test_harness.h index 35ddd2209b9..f4f1c7a8c3a 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/test_harness.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/test_harness.h @@ -6,16 +6,13 @@ #include <cinttypes> /* Include various wiredtiger libs. */ +#include "configuration_settings.h" #include "wiredtiger.h" #include "wt_internal.h" -#include "configuration_settings.h" - namespace test_harness { class test { public: - configuration *_configuration; - static const char *_name; /* * All tests will implement this initially, the return value from it will indicate whether the * test was successful or not. @@ -30,7 +27,12 @@ class test { ~test() { delete _configuration; + _configuration = nullptr; } + + configuration *_configuration = nullptr; + static const char *_name; + static const char *_default_config; }; } // namespace test_harness diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h new file mode 100644 index 00000000000..30771db14a7 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload_generator.h @@ -0,0 +1,166 @@ +/* Include guard. */ +#ifndef WORKLOAD_GENERATOR_H +#define WORKLOAD_GENERATOR_H + +#include <cstdint> +#include <map> +#include <vector> + +extern "C" { +#include "test_util.h" +} + +#include "api_const.h" +#include "configuration_settings.h" +#include "random_generator.h" + +#define DEBUG_ERROR 0 +#define DEBUG_INFO 1 + +namespace test_harness { +class workload_generator { + public: + workload_generator(test_harness::configuration *configuration) + { + _configuration = configuration; + } + + ~workload_generator() + { + if (_session != nullptr) { + if (_session->close(_session, NULL) != 0) + /* Failing to close session is not blocking. */ + debug_info("Failed to close session, shutting down uncleanly", DEBUG_ERROR); + _session = nullptr; + } + + if (_conn != nullptr) { + if (_conn->close(_conn, NULL) != 0) + /* Failing to close connection is not blocking. */ + debug_info("Failed to close connection, shutting down uncleanly", DEBUG_ERROR); + _conn = nullptr; + } + } + + /* + * Function that performs the following steps using the configuration that is defined by the + * test: + * - Create the working dir. + * - Open a connection. + * - Open a session. + * - Create n collections as per the configuration. + * - Open a cursor on each collection. + * - Insert m key/value pairs in each collection. Values are random strings which size is + * defined by the configuration. + */ + int + load(const char *home = DEFAULT_DIR) + { + WT_CURSOR *cursor; + int64_t collection_count, key_count, value_size; + std::string collection_name; + + cursor = nullptr; + collection_count = key_count = value_size = 0; + collection_name = ""; + + /* Create the working dir. */ + testutil_make_work_dir(home); + + /* Open connection. */ + WT_RET(wiredtiger_open(home, NULL, test_harness::CONNECTION_CREATE, &_conn)); + + /* Open session. */ + WT_RET(_conn->open_session(_conn, NULL, NULL, &_session)); + + /* Create n collections as per the configuration and store each collection name. */ + WT_RET(_configuration->get_int(test_harness::COLLECTION_COUNT, collection_count)); + for (int i = 0; i < collection_count; ++i) { + collection_name = "table:collection" + std::to_string(i); + WT_RET(_session->create(_session, collection_name.c_str(), DEFAULT_TABLE_SCHEMA)); + _collection_names.push_back(collection_name); + } + debug_info(std::to_string(collection_count) + " collections created", DEBUG_INFO); + + /* Open a cursor on each collection and use the configuration to insert key/value pairs. */ + WT_RET(_configuration->get_int(test_harness::KEY_COUNT, key_count)); + WT_RET(_configuration->get_int(test_harness::VALUE_SIZE, value_size)); + for (const auto &collection_name : _collection_names) { + /* WiredTiger lets you open a cursor on a collection using the same pointer. When a + * session is closed, WiredTiger APIs close the cursors too. */ + WT_RET(_session->open_cursor(_session, collection_name.c_str(), NULL, NULL, &cursor)); + for (size_t j = 0; j < key_count; ++j) { + cursor->set_key(cursor, j); + /* Generation of a random string value using the size defined in the test + * configuration. */ + std::string generated_value = + random_generator::random_generator::get_instance()->generate_string(value_size); + cursor->set_value(cursor, generated_value.c_str()); + WT_RET(cursor->insert(cursor)); + } + } + debug_info(std::to_string(collection_count) + " key/value inserted", DEBUG_INFO); + return (0); + } + + /* Do the work of the main part of the workload. */ + int + run() + { + /* Empty until thread management lib is implemented. */ + return (0); + } + + /* WiredTiger APIs wrappers for single operations. */ + int + insert(WT_CURSOR *cursor, const char *value) + { + if (cursor == nullptr) + throw std::invalid_argument("Failed to call insert, invalid cursor"); + return (cursor->insert(cursor)); + } + + int + search(WT_CURSOR *cursor) + { + if (cursor == nullptr) + throw std::invalid_argument("Failed to call search, invalid cursor"); + return (cursor->search(cursor)); + } + + int + search_near(WT_CURSOR *cursor, int *exact) + { + if (cursor == nullptr) + throw std::invalid_argument("Failed to call search_near, invalid cursor"); + return (cursor->search_near(cursor, exact)); + } + + int + update(WT_CURSOR *cursor) + { + if (cursor == nullptr) + throw std::invalid_argument("Failed to call update, invalid cursor"); + return (cursor->update(cursor)); + } + + /* Trace level that is set in the test configuration for debugging purpose. */ + static int64_t _trace_level; + + private: + /* Used to print out traces for debugging purpose. */ + static void + debug_info(const std::string &str, int64_t trace_level) + { + if (_trace_level >= trace_level) + std::cout << str << std::endl; + } + + std::vector<std::string> _collection_names; + test_harness::configuration *_configuration = nullptr; + WT_CONNECTION *_conn = nullptr; + WT_SESSION *_session = nullptr; +}; +} // namespace test_harness + +#endif diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx index c58ed9a1119..484f14845b0 100644..100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx @@ -1,35 +1,79 @@ #include <iostream> #include <cstdlib> + #include "test_harness/test_harness.h" +#include "test_harness/workload_generator.h" class poc_test : public test_harness::test { public: - int run() { - WT_CONNECTION *conn; - int ret = 0; - /* Setup basic test directory. */ - const char *default_dir = "WT_TEST"; - - /* - * Csuite tests utilise a test_util.h command to make their directory, currently that doesn't - * compile under c++ and some extra work will be needed to make it work. Its unclear if the - * test framework will use test_util.h yet. - */ - const char *mkdir_cmd = "mkdir WT_TEST"; - ret = system(mkdir_cmd); - if (ret != 0) - return (ret); - - ret = wiredtiger_open(default_dir, NULL, "create,cache_size=1G", &conn); - return (ret); + poc_test(const char *config, int64_t trace_level) : test(config) + { + test_harness::workload_generator::_trace_level = trace_level; + _wl = new test_harness::workload_generator(_configuration); + } + + ~poc_test() + { + delete _wl; + _wl = nullptr; } - poc_test(const char *config) : test(config) {} + int + run() + { + int return_code = _wl->load(); + if (return_code != 0) + throw std::runtime_error( + "Load stage failed with error code: " + std::to_string(return_code)); + return_code = _wl->run(); + if (return_code != 0) + throw std::runtime_error( + "Run stage failed with error code: " + std::to_string(return_code)); + return return_code; + } + + private: + test_harness::workload_generator *_wl = nullptr; }; const char *poc_test::test::_name = "poc_test"; +const char *poc_test::test::_default_config = "collection_count=2,key_count=5,value_size=20"; +int64_t test_harness::workload_generator::_trace_level = 0; + +int +main(int argc, char *argv[]) +{ + std::string cfg = ""; + int64_t trace_level = 0; + + // Parse args + // -C : Configuration + // -t : Trace level + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-C") == 0) { + if (i + 1 < argc) { + cfg = argv[++i]; + } else { + throw std::invalid_argument("No value given for option " + std::string(argv[i])); + std::cout << "No value given for option " << argv[i] << std::endl; + } + } else if (strcmp(argv[i], "-t") == 0) { + if (i + 1 < argc) { + trace_level = std::stoi(argv[++i]); + } else { + throw std::invalid_argument("No value given for option " + std::string(argv[i])); + } + } + } + + // Check if default configuration should be used + if (cfg.compare("") == 0) { + std::cout << "Using default configuration" << std::endl; + cfg = poc_test::test::_default_config; + } + + std::cout << "Configuration\t:" << cfg << std::endl; + std::cout << "Tracel level\t:" << trace_level << std::endl; -int main(int argc, char *argv[]) { - const char *cfg = "collection_count=1,key_size=5"; - return poc_test(cfg).run(); + return (poc_test(cfg.c_str(), trace_level).run()); } diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py new file mode 100644 index 00000000000..19c18c35cbd --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wtscenario, wttest +from wiredtiger import stat +from wtdataset import SimpleDataSet + +# test_tiered04.py +# Basic tiered storage API test. +class test_tiered04(wttest.WiredTigerTestCase): + uri = "table:test_tiered04" + + auth_token = "test_token" + extension_name = "test" + retention = 600 + def conn_config(self): + return \ + 'statistics=(fast),' + \ + 'tiered_storage=(enabled,local_retention=%d,' % self.retention + \ + 'name=%s,' % self.extension_name + \ + 'auth_token=%s)' % self.auth_token + + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + + # Test calling the flush_tier API. + def test_tiered(self): + self.session.create(self.uri, 'key_format=S') + # The retention is in minutes. The stat is in seconds. + retain = self.get_stat(stat.conn.tiered_retention) + self.assertEqual(retain, self.retention) + self.session.flush_tier(None) + self.session.flush_tier('force=true') + calls = self.get_stat(stat.conn.flush_tier) + self.assertEqual(calls, 2) + new = self.retention * 2 + config = 'tiered_storage=(local_retention=%d)' % new + self.conn.reconfigure(config) + self.session.flush_tier(None) + retain = self.get_stat(stat.conn.tiered_retention) + calls = self.get_stat(stat.conn.flush_tier) + self.assertEqual(retain, new) + self.assertEqual(calls, 3) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered05.py b/src/third_party/wiredtiger/test/suite/test_tiered05.py new file mode 100644 index 00000000000..e0e1502f542 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_tiered05.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wtscenario, wttest +from wiredtiger import stat + +# test_tiered05.py +# Basic tiered storage API test. +class test_tiered05(wttest.WiredTigerTestCase): + uri = "table:test_tiered05" + + auth_token = "test_token" + extension_name = "test" + def conn_config(self): + return \ + 'statistics=(fast),' + \ + 'tiered_manager=(wait=10),' + \ + 'tiered_storage=(enabled,object_target_size=20M,' + \ + 'name=%s,' % self.extension_name + \ + 'auth_token=%s)' % self.auth_token + + # Test calling the flush_tier API. + def test_tiered(self): + self.session.create(self.uri, 'key_format=S') + msg = "/storage manager thread is configured/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(self.session.flush_tier(None), 0), msg) + +if __name__ == '__main__': + wttest.run() |