Import wiredtiger: 0c6ba8d8be02dd34a46c3e9533971f1739b6ad8e from branch mongodb-4.2

ref: 92719d6bc9..0c6ba8d8be
for: 4.1.9

WT-4053       Add statistic for data handle size
WT-4296       Add aggressive mode to sweep server
WT-4313       Update query_timestamp documentation for last_checkpoint and recovery timestamp
WT-4369       random_directio shouldn't assume children threads wrote any data
WT-4433       Enable dynamic compression for snappy
WT-4447       Add prototype implementation allowing limiting of IO per subsystem
WT-4484       workgen: wtperf.py should store python intermediate files in home directory
WT-4489       Store correct timestamp in cells on disk
WT-4492       Automate testing of versioned data format
WT-4501       Fix a race condition opening/closing the shared cache
WT-4533       Upgrade to v3 toolchain for Evergreen tests
WT-4546       Add workgen improvements and test files to simulate read and write "storms"
WT-4547       Consolidate capacity and block manager bytes written
WT-4557       Remove error_prefix from base configuration file, set error_prefix in wt command
WT-4559       Create a marker file when a test leaves a database corrupted
WT-4561       WiredTiger LSM chunk drop can race with hot backups
WT-4562       Coverity: possible divide by zero conn_capacity:434
WT-4569       WT_CONN_DATA_CORRUPTION incorrectly set in Btree verify code.
WT-4576       Coverity warning in conn_capacity.c

1 files changed, 92 insertions, 67 deletions

diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index 32939a97c72..90b1dc023ec 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -93,6 +93,12 @@ static const WT_CONFIG_CHECK
 };
 
 static const WT_CONFIG_CHECK
+    confchk_wiredtiger_open_io_capacity_subconfigs[] = {
+	{ "total", "int", NULL, "min=0,max=1TB", NULL, 0 },
+	{ NULL, NULL, NULL, NULL, NULL, 0 }
+};
+
+static const WT_CONFIG_CHECK
     confchk_WT_CONNECTION_reconfigure_log_subconfigs[] = {
 	{ "archive", "boolean", NULL, NULL, NULL, 0 },
 	{ "os_cache_dirty_pct", "int",
@@ -170,6 +176,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = {
 	{ "file_manager", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_file_manager_subconfigs, 3 },
+	{ "io_capacity", "category",
+	    NULL, NULL,
+	    confchk_wiredtiger_open_io_capacity_subconfigs, 1 },
 	{ "log", "category",
 	    NULL, NULL,
 	    confchk_WT_CONNECTION_reconfigure_log_subconfigs, 4 },
@@ -191,9 +200,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = {
 	    NULL, NULL,
 	    confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\","
-	    "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
-	    "\"split_6\",\"split_7\",\"split_8\"]",
+	    NULL, "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
+	    "\"lookaside_sweep_race\",\"split_1\",\"split_2\",\"split_3\","
+	    "\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
 	    NULL, 0 },
 	{ "verbose", "list",
 	    NULL, "choices=[\"api\",\"block\",\"checkpoint\","
@@ -876,6 +885,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = {
 	    confchk_wiredtiger_open_file_manager_subconfigs, 3 },
 	{ "hazard_max", "int", NULL, "min=15", NULL, 0 },
 	{ "in_memory", "boolean", NULL, NULL, NULL, 0 },
+	{ "io_capacity", "category",
+	    NULL, NULL,
+	    confchk_wiredtiger_open_io_capacity_subconfigs, 1 },
 	{ "log", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_log_subconfigs, 9 },
@@ -904,9 +916,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_statistics_log_subconfigs, 6 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\","
-	    "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
-	    "\"split_6\",\"split_7\",\"split_8\"]",
+	    NULL, "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
+	    "\"lookaside_sweep_race\",\"split_1\",\"split_2\",\"split_3\","
+	    "\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
 	    NULL, 0 },
 	{ "transaction_sync", "category",
 	    NULL, NULL,
@@ -982,6 +994,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = {
 	    confchk_wiredtiger_open_file_manager_subconfigs, 3 },
 	{ "hazard_max", "int", NULL, "min=15", NULL, 0 },
 	{ "in_memory", "boolean", NULL, NULL, NULL, 0 },
+	{ "io_capacity", "category",
+	    NULL, NULL,
+	    confchk_wiredtiger_open_io_capacity_subconfigs, 1 },
 	{ "log", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_log_subconfigs, 9 },
@@ -1010,9 +1025,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_statistics_log_subconfigs, 6 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\","
-	    "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
-	    "\"split_6\",\"split_7\",\"split_8\"]",
+	    NULL, "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
+	    "\"lookaside_sweep_race\",\"split_1\",\"split_2\",\"split_3\","
+	    "\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
 	    NULL, 0 },
 	{ "transaction_sync", "category",
 	    NULL, NULL,
@@ -1085,6 +1100,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_file_manager_subconfigs, 3 },
 	{ "hazard_max", "int", NULL, "min=15", NULL, 0 },
+	{ "io_capacity", "category",
+	    NULL, NULL,
+	    confchk_wiredtiger_open_io_capacity_subconfigs, 1 },
 	{ "log", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_log_subconfigs, 9 },
@@ -1113,9 +1131,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_statistics_log_subconfigs, 6 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\","
-	    "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
-	    "\"split_6\",\"split_7\",\"split_8\"]",
+	    NULL, "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
+	    "\"lookaside_sweep_race\",\"split_1\",\"split_2\",\"split_3\","
+	    "\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
 	    NULL, 0 },
 	{ "transaction_sync", "category",
 	    NULL, NULL,
@@ -1186,6 +1204,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_file_manager_subconfigs, 3 },
 	{ "hazard_max", "int", NULL, "min=15", NULL, 0 },
+	{ "io_capacity", "category",
+	    NULL, NULL,
+	    confchk_wiredtiger_open_io_capacity_subconfigs, 1 },
 	{ "log", "category",
 	    NULL, NULL,
 	    confchk_wiredtiger_open_log_subconfigs, 9 },
@@ -1214,9 +1235,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = {
 	    NULL, NULL,
 	    confchk_wiredtiger_open_statistics_log_subconfigs, 6 },
 	{ "timing_stress_for_test", "list",
-	    NULL, "choices=[\"checkpoint_slow\",\"lookaside_sweep_race\","
-	    "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\","
-	    "\"split_6\",\"split_7\",\"split_8\"]",
+	    NULL, "choices=[\"aggressive_sweep\",\"checkpoint_slow\","
+	    "\"lookaside_sweep_race\",\"split_1\",\"split_2\",\"split_3\","
+	    "\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]",
 	    NULL, 0 },
 	{ "transaction_sync", "category",
 	    NULL, NULL,
@@ -1294,15 +1315,15 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "eviction_checkpoint_target=1,eviction_dirty_target=5,"
 	  "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
 	  ",file_manager=(close_handle_minimum=250,close_idle_time=30,"
-	  "close_scan_interval=10),log=(archive=true,os_cache_dirty_pct=0,"
-	  "prealloc=true,zero_fill=false),lsm_manager=(merge=true,"
-	  "worker_thread_max=4),lsm_merge=true,"
+	  "close_scan_interval=10),io_capacity=(total=0),log=(archive=true,"
+	  "os_cache_dirty_pct=0,prealloc=true,zero_fill=false),"
+	  "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true,"
 	  "operation_tracking=(enabled=false,path=\".\"),"
 	  "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB),"
 	  "statistics=none,statistics_log=(json=false,on_close=false,"
 	  "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
 	  "timing_stress_for_test=,verbose=",
-	  confchk_WT_CONNECTION_reconfigure, 23
+	  confchk_WT_CONNECTION_reconfigure, 24
 	},
 	{ "WT_CONNECTION.rollback_to_stable",
 	  "",
@@ -1548,19 +1569,20 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "eviction_target=80,eviction_trigger=95,exclusive=false,"
 	  "extensions=,file_extend=,file_manager=(close_handle_minimum=250,"
 	  "close_idle_time=30,close_scan_interval=10),hazard_max=1000,"
-	  "in_memory=false,log=(archive=true,compressor=,enabled=false,"
-	  "file_max=100MB,os_cache_dirty_pct=0,path=\".\",prealloc=true,"
-	  "recover=on,zero_fill=false),lsm_manager=(merge=true,"
-	  "worker_thread_max=4),lsm_merge=true,mmap=true,multiprocess=false"
-	  ",operation_tracking=(enabled=false,path=\".\"),readonly=false,"
-	  "salvage=false,session_max=100,session_scratch_max=2MB,"
-	  "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0,"
-	  "reserve=0,size=500MB),statistics=none,statistics_log=(json=false"
-	  ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\""
-	  ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false"
-	  ",method=fsync),use_environment=true,use_environment_priv=false,"
+	  "in_memory=false,io_capacity=(total=0),log=(archive=true,"
+	  "compressor=,enabled=false,file_max=100MB,os_cache_dirty_pct=0,"
+	  "path=\".\",prealloc=true,recover=on,zero_fill=false),"
+	  "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true,"
+	  "mmap=true,multiprocess=false,operation_tracking=(enabled=false,"
+	  "path=\".\"),readonly=false,salvage=false,session_max=100,"
+	  "session_scratch_max=2MB,session_table_cache=true,"
+	  "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB),"
+	  "statistics=none,statistics_log=(json=false,on_close=false,"
+	  "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
+	  "timing_stress_for_test=,transaction_sync=(enabled=false,"
+	  "method=fsync),use_environment=true,use_environment_priv=false,"
 	  "verbose=,write_through=",
-	  confchk_wiredtiger_open, 47
+	  confchk_wiredtiger_open, 48
 	},
 	{ "wiredtiger_open_all",
 	  "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1"
@@ -1575,19 +1597,20 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "eviction_target=80,eviction_trigger=95,exclusive=false,"
 	  "extensions=,file_extend=,file_manager=(close_handle_minimum=250,"
 	  "close_idle_time=30,close_scan_interval=10),hazard_max=1000,"
-	  "in_memory=false,log=(archive=true,compressor=,enabled=false,"
-	  "file_max=100MB,os_cache_dirty_pct=0,path=\".\",prealloc=true,"
-	  "recover=on,zero_fill=false),lsm_manager=(merge=true,"
-	  "worker_thread_max=4),lsm_merge=true,mmap=true,multiprocess=false"
-	  ",operation_tracking=(enabled=false,path=\".\"),readonly=false,"
-	  "salvage=false,session_max=100,session_scratch_max=2MB,"
-	  "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0,"
-	  "reserve=0,size=500MB),statistics=none,statistics_log=(json=false"
-	  ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\""
-	  ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false"
-	  ",method=fsync),use_environment=true,use_environment_priv=false,"
+	  "in_memory=false,io_capacity=(total=0),log=(archive=true,"
+	  "compressor=,enabled=false,file_max=100MB,os_cache_dirty_pct=0,"
+	  "path=\".\",prealloc=true,recover=on,zero_fill=false),"
+	  "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true,"
+	  "mmap=true,multiprocess=false,operation_tracking=(enabled=false,"
+	  "path=\".\"),readonly=false,salvage=false,session_max=100,"
+	  "session_scratch_max=2MB,session_table_cache=true,"
+	  "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB),"
+	  "statistics=none,statistics_log=(json=false,on_close=false,"
+	  "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
+	  "timing_stress_for_test=,transaction_sync=(enabled=false,"
+	  "method=fsync),use_environment=true,use_environment_priv=false,"
 	  "verbose=,version=(major=0,minor=0),write_through=",
-	  confchk_wiredtiger_open_all, 48
+	  confchk_wiredtiger_open_all, 49
 	},
 	{ "wiredtiger_open_basecfg",
 	  "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1"
@@ -1601,18 +1624,19 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
 	  ",extensions=,file_extend=,file_manager=(close_handle_minimum=250"
 	  ",close_idle_time=30,close_scan_interval=10),hazard_max=1000,"
-	  "log=(archive=true,compressor=,enabled=false,file_max=100MB,"
-	  "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on,"
-	  "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4),"
-	  "lsm_merge=true,mmap=true,multiprocess=false,"
-	  "operation_tracking=(enabled=false,path=\".\"),readonly=false,"
-	  "salvage=false,session_max=100,session_scratch_max=2MB,"
-	  "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0,"
-	  "reserve=0,size=500MB),statistics=none,statistics_log=(json=false"
-	  ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\""
-	  ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false"
-	  ",method=fsync),verbose=,version=(major=0,minor=0),write_through=",
-	  confchk_wiredtiger_open_basecfg, 42
+	  "io_capacity=(total=0),log=(archive=true,compressor=,"
+	  "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\","
+	  "prealloc=true,recover=on,zero_fill=false),"
+	  "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true,"
+	  "mmap=true,multiprocess=false,operation_tracking=(enabled=false,"
+	  "path=\".\"),readonly=false,salvage=false,session_max=100,"
+	  "session_scratch_max=2MB,session_table_cache=true,"
+	  "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB),"
+	  "statistics=none,statistics_log=(json=false,on_close=false,"
+	  "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
+	  "timing_stress_for_test=,transaction_sync=(enabled=false,"
+	  "method=fsync),verbose=,version=(major=0,minor=0),write_through=",
+	  confchk_wiredtiger_open_basecfg, 43
 	},
 	{ "wiredtiger_open_usercfg",
 	  "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1"
@@ -1626,18 +1650,19 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95"
 	  ",extensions=,file_extend=,file_manager=(close_handle_minimum=250"
 	  ",close_idle_time=30,close_scan_interval=10),hazard_max=1000,"
-	  "log=(archive=true,compressor=,enabled=false,file_max=100MB,"
-	  "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on,"
-	  "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4),"
-	  "lsm_merge=true,mmap=true,multiprocess=false,"
-	  "operation_tracking=(enabled=false,path=\".\"),readonly=false,"
-	  "salvage=false,session_max=100,session_scratch_max=2MB,"
-	  "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0,"
-	  "reserve=0,size=500MB),statistics=none,statistics_log=(json=false"
-	  ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\""
-	  ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false"
-	  ",method=fsync),verbose=,write_through=",
-	  confchk_wiredtiger_open_usercfg, 41
+	  "io_capacity=(total=0),log=(archive=true,compressor=,"
+	  "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\","
+	  "prealloc=true,recover=on,zero_fill=false),"
+	  "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true,"
+	  "mmap=true,multiprocess=false,operation_tracking=(enabled=false,"
+	  "path=\".\"),readonly=false,salvage=false,session_max=100,"
+	  "session_scratch_max=2MB,session_table_cache=true,"
+	  "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB),"
+	  "statistics=none,statistics_log=(json=false,on_close=false,"
+	  "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0),"
+	  "timing_stress_for_test=,transaction_sync=(enabled=false,"
+	  "method=fsync),verbose=,write_through=",
+	  confchk_wiredtiger_open_usercfg, 42
 	},
 	{ NULL, NULL, NULL, 0 }
 };