Import wiredtiger: f90baa428bfefb17afddd8f8aa72e8d34cf761e2 from branch mongodb-5.0

ref: 5641affc86..f90baa428b
for: 4.9.0

WT-6513       Update history store configuration to make overflow keys unlikely
WT-6850       Create and add config parsing for the workload generator class
WT-6950       Create test coverage documentation landing page
WT-7087       API for tiered storage
WT-7211       Added missing return statement in stress testing framework

5 files changed, 112 insertions, 11 deletions

diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 74c76adc5c7..c42cf702495 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -721,6 +721,24 @@ connection_runtime_config = [
             this will update the value if one is already set''',
             min='1MB', max='10TB')
         ]),
+    Config('tiered_manager', '', r'''
+        tiered storage manager configuration options''',
+        type='category', undoc=True, subconfig=[
+            Config('threads_max', '8', r'''
+                maximum number of threads WiredTiger will start to help manage
+                tiered storage maintenance. Each worker thread uses a session
+                from the configured session_max''',
+                min=1, max=20),
+            Config('threads_min', '1', r'''
+                minimum number of threads WiredTiger will start to help manage
+                tiered storage maintenance.''',
+                min=1, max=20),
+            Config('wait', '0', r'''
+                seconds to wait between each periodic housekeeping of
+                tiered storage. Setting this value above 0 configures periodic
+                management inside WiredTiger''',
+                min='0', max='100000'),
+            ]),
     Config('statistics', 'none', r'''
         Maintain database statistics, which may impact performance.
         Choosing "all" maintains all statistics regardless of cost,
@@ -910,6 +928,42 @@ connection_reconfigure_statistics_log_configuration = [
         type='category', subconfig=
         statistics_log_configuration_common)
 ]
+
+tiered_storage_configuration_common = [
+    Config('auth_token', '', r'''
+        authentication token string'''),
+    Config('local_retention', '300', r'''
+        time in seconds to retain data on tiered storage on the local tier for
+        faster read access''',
+        min='0', max='10000'),
+    Config('object_target_size', '10M', r'''
+        the approximate size of objects before creating them on the
+        tiered storage tier''',
+        min='100K', max='10TB'),
+]
+connection_reconfigure_tiered_storage_configuration = [
+    Config('tiered_storage', '', r'''
+        enable tiered storage. Enabling tiered storage may use one session from the
+        configured session_max''',
+        type='category', subconfig=
+        tiered_storage_configuration_common)
+]
+wiredtiger_open_tiered_storage_configuration = [
+    Config('tiered_storage', '', r'''
+        enable tiered storage. Enabling tiered storage may use one session from the
+        configured session_max''',
+        type='category', undoc=True, subconfig=
+        tiered_storage_configuration_common + [
+        Config('enabled', 'false', r'''
+            enable tiered storage subsystem''',
+            type='boolean'),
+        Config('name', 'none', r'''
+            Permitted values are \c "none"
+            or custom storage name created with
+            WT_CONNECTION::add_tiered_storage'''),
+    ]),
+]
+
 wiredtiger_open_statistics_log_configuration = [
     Config('statistics_log', '', r'''
         log any statistics the database is configured to maintain,
@@ -950,6 +1004,7 @@ wiredtiger_open_common =\
     connection_runtime_config +\
     wiredtiger_open_compatibility_configuration +\
     wiredtiger_open_log_configuration +\
+    wiredtiger_open_tiered_storage_configuration +\
     wiredtiger_open_statistics_log_configuration + [
     Config('buffer_alignment', '-1', r'''
         in-memory alignment (in bytes) for buffers used for I/O.  The
@@ -1448,6 +1503,13 @@ methods = {
         files''',
         type='boolean'),
 ]),
+
+'WT_SESSION.flush_tier' : Method([
+    Config('force', 'false', r'''
+        force sharing of all data''',
+        type='boolean'),
+]),
+
 'WT_SESSION.strerror' : Method([]),
 'WT_SESSION.transaction_sync' : Method([
     Config('timeout_ms', '1200000', # !!! Must match WT_SESSION_BG_SYNC_MSEC
@@ -1685,6 +1747,7 @@ methods = {
     connection_reconfigure_compatibility_configuration +\
     connection_reconfigure_log_configuration +\
     connection_reconfigure_statistics_log_configuration +\
+    connection_reconfigure_tiered_storage_configuration +\
     connection_runtime_config
 ),
 'WT_CONNECTION.set_file_system' : Method([]),
diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist
index 39b85746b6c..35ff295c206 100644
--- a/src/third_party/wiredtiger/dist/filelist
+++ b/src/third_party/wiredtiger/dist/filelist
@@ -74,6 +74,7 @@ src/conn/conn_open.c
 src/conn/conn_reconfig.c
 src/conn/conn_stat.c
 src/conn/conn_sweep.c
+src/conn/conn_tiered.c
 src/cursor/cur_backup.c
 src/cursor/cur_backup_incr.c
 src/cursor/cur_bulk.c
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index f9f2b67712f..c5b3fe73c20 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -503,6 +503,7 @@ aspell
 assertfmt
 async
 atomicity
+auth
 autocommit
 autoconf
 automake
diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py
index 77808db1c9d..97c20322319 100644
--- a/src/third_party/wiredtiger/dist/stat_data.py
+++ b/src/third_party/wiredtiger/dist/stat_data.py
@@ -104,6 +104,10 @@ class SessionOpStat(Stat):
     prefix = 'session'
     def __init__(self, name, desc, flags=''):
         Stat.__init__(self, name, SessionOpStat.prefix, desc, flags)
+class StorageStat(Stat):
+    prefix = 'session'
+    def __init__(self, name, desc, flags=''):
+        Stat.__init__(self, name, SessionOpStat.prefix, desc, flags)
 class ThreadStat(Stat):
     prefix = 'thread-state'
     def __init__(self, name, desc, flags=''):
@@ -463,6 +467,8 @@ connection_stats = [
     # Reconciliation statistics
     ##########################################
     RecStat('rec_maximum_seconds', 'maximum seconds spent in a reconciliation call', 'no_clear,no_scale,size'),
+    RecStat('rec_overflow_key_internal', 'internal-page overflow keys'),
+    RecStat('rec_overflow_key_leaf', 'leaf-page overflow keys'),
     RecStat('rec_pages_with_prepare', 'page reconciliation calls that resulted in values with prepared transaction metadata'),
     RecStat('rec_pages_with_ts', 'page reconciliation calls that resulted in values with timestamps'),
     RecStat('rec_pages_with_txn', 'page reconciliation calls that resulted in values with transaction ids'),
@@ -855,6 +861,12 @@ conn_dsrc_stats = [
     RecStat('rec_time_window_stop_txn', 'records written including a stop transaction ID'),
 
     ##########################################
+    # Tiered storage statistics
+    ##########################################
+    StorageStat('flush_tier', 'flush_tier operation calls'),
+    StorageStat('tiered_retention', 'tiered storage local retention time (secs)', 'no_clear,no_scale,size'),
+
+    ##########################################
     # Transaction statistics
     ##########################################
     TxnStat('txn_checkpoint_obsolete_applied', 'transaction checkpoints due to obsolete pages'),
diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py
index ac46cf55bc2..3fedb10b5d4 100644
--- a/src/third_party/wiredtiger/dist/test_data.py
+++ b/src/third_party/wiredtiger/dist/test_data.py
@@ -41,16 +41,40 @@ class Config:
 
     def __ge__(self, other):
         return self.name >= other.name
+
+key_config=[
+    Config('key_size', 0, r'''
+        The size of the keys to be created''', min=0, max=10000),
+]
+
+value_config = [
+    Config('value_size', 0, r'''
+        The size of the values to be created''', min=0, max=10000),
+]
+
+scale_config = [
+    Config('collection_count', 1, r'''
+        The number of colections the workload generator operates over''', min=0, max=200000),
+    Config('key_count', 0, r'''
+        The number of keys to be operated on per colection''', min=0, max=1000000),
+]
+
+load_config = key_config + value_config + scale_config
+
+workload_config = [
+    Config('read_threads', 0, r'''
+        The number of threads performing read operations''', min=0, max=100),
+    Config('insert_threads', 0, r'''
+        The number of threads performing insert operations''',min=0, max=20),
+    Config('insert_config',0, r'''
+        The definition of the record being inserted''',
+        subconfig=load_config),
+    Config('update_threads', 0, r'''
+        The number of threads performing update operations''',min=0, max=20),
+    Config('update_config',0,r''',
+        The definition of the record being updated''', subconfig=load_config)
+]
+
 methods = {
-'poc_test' : Method([
-    Config('collection_count', '1', r'''
-        the number of collections to create for testing''',
-        min='1', max='10'),
-    Config('key_size', '10', r'''
-        the size of the keys to be created in bytes''',
-        min='1', max='10000'),
-    Config('values', 'first', r'''
-        The value that each key will be populated with, used an example string configuration''',
-        choices=['first', 'second', 'third'])
-]),
+'poc_test' : Method(load_config + workload_config),
 }