diff options
author | Luke Chen <luke.chen@mongodb.com> | 2023-03-20 15:23:51 +1100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-03-20 04:41:05 +0000 |
commit | afcc1b1dabab5f4372ce3ff5c9133e56e9852704 (patch) | |
tree | c0c74972f06c9552bddea552f380169784da9259 | |
parent | 3d472283f006afe6724afc79b66b7c9d17f4ac5f (diff) | |
download | mongo-afcc1b1dabab5f4372ce3ff5c9133e56e9852704.tar.gz |
Import wiredtiger: c5b5e47d2c3023a41782be617dad72aa13ffa86c from branch mongodb-4.4
ref: 4761fa3580..c5b5e47d2c
for: 4.4.20
WT-9251 Perform log flush before the metadata checkpoint
-rw-r--r-- | src/third_party/wiredtiger/dist/api_data.py | 12 | ||||
-rw-r--r-- | src/third_party/wiredtiger/dist/stat_data.py | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/import.data | 2 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/config/config_def.c | 72 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/conn/conn_api.c | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/conn/conn_log.c | 20 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/connection.h | 34 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/stat.h | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/wiredtiger.in | 74 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/support/stat.c | 3 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/txn/txn_ckpt.c | 14 | ||||
-rwxr-xr-x | src/third_party/wiredtiger/test/suite/test_rollback_to_stable35.py | 182 |
12 files changed, 326 insertions, 90 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 4a4b4d16c5e..b6ac3b1643d 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -801,9 +801,10 @@ connection_runtime_config = [ type='list', undoc=True, choices=[ 'aggressive_sweep', 'backup_rename', 'checkpoint_reserved_txnid_delay', 'checkpoint_slow', - 'failpoint_history_store_delete_key_from_ts', 'history_store_checkpoint_delay', - 'history_store_search', 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', - 'split_2', 'split_3', 'split_4', 'split_5', 'split_6', 'split_7']), + 'checkpoint_stop', 'failpoint_history_store_delete_key_from_ts', + 'history_store_checkpoint_delay', 'history_store_search', 'history_store_sweep_race', + 'prepare_checkpoint_delay', 'split_1', 'split_2', 'split_3', 'split_4', 'split_5', + 'split_6', 'split_7']), Config('verbose', '[]', r''' enable messages for various events. Options are given as a list, such as <code>"verbose=[evictserver,read]"</code>''', @@ -926,6 +927,11 @@ wiredtiger_open_log_configuration = [ the maximum size of log files''', min='100KB', # !!! Must match WT_LOG_FILE_MIN max='2GB'), # !!! Must match WT_LOG_FILE_MAX + Config('force_write_wait', '0', r''' + enable code that interrupts the usual timing of flushing the log from + the internal log server thread with a goal of uncovering race conditions. + This option is intended for use with internal stress testing of WiredTiger.''', + min='1', max='60', undoc=True), Config('path', '"."', r''' the name of a directory into which log files are written. The directory must already exist. If the value is not an absolute path, diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index c609c65b56c..ce970b3305b 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -549,6 +549,7 @@ conn_stats = [ TxnStat('txn_checkpoint_time_min', 'transaction checkpoint min time (msecs)', 'no_clear,no_scale'), TxnStat('txn_checkpoint_time_recent', 'transaction checkpoint most recent time (msecs)', 'no_clear,no_scale'), TxnStat('txn_checkpoint_time_total', 'transaction checkpoint total time (msecs)', 'no_clear,no_scale'), + TxnStat('txn_checkpoint_stop_stress_active', 'transaction checkpoint stop timing stress active', 'no_clear,no_scale'), TxnStat('txn_commit', 'transactions committed'), TxnStat('txn_fail_cache', 'transaction failures due to history store'), TxnStat('txn_pinned_checkpoint_range', 'transaction range of IDs currently pinned by a checkpoint', 'no_clear,no_scale'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index d9564c5a8d2..d89f190d17f 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "4761fa3580dc3c9a90bfdeec4c2f6c6e83736d8d" + "commit": "c5b5e47d2c3023a41782be617dad72aa13ffa86c" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index e128f69a4da..f7219dd3b19 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -137,6 +137,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," + "\"checkpoint_stop\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -787,6 +788,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_log_subconfigs[] = { {"archive", "boolean", NULL, NULL, NULL, 0}, {"compressor", "string", NULL, NULL, NULL, 0}, {"enabled", "boolean", NULL, NULL, NULL, 0}, {"file_max", "int", NULL, "min=100KB,max=2GB", NULL, 0}, + {"force_write_wait", "int", NULL, "min=1,max=60", NULL, 0}, {"os_cache_dirty_pct", "int", NULL, "min=0,max=100", NULL, 0}, {"path", "string", NULL, NULL, NULL, 0}, {"prealloc", "boolean", NULL, NULL, NULL, 0}, {"recover", "string", NULL, "choices=[\"error\",\"on\"]", NULL, 0}, @@ -842,7 +844,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"history_store", "category", NULL, NULL, confchk_wiredtiger_open_history_store_subconfigs, 1}, {"in_memory", "boolean", NULL, NULL, NULL, 0}, {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, - {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, + {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 10}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, {"mmap", "boolean", NULL, NULL, NULL, 0}, {"mmap_all", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, @@ -864,6 +866,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," + "\"checkpoint_stop\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -922,7 +925,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"history_store", "category", NULL, NULL, confchk_wiredtiger_open_history_store_subconfigs, 1}, {"in_memory", "boolean", NULL, NULL, NULL, 0}, {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, - {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, + {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 10}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, {"mmap", "boolean", NULL, NULL, NULL, 0}, {"mmap_all", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, @@ -944,6 +947,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," + "\"checkpoint_stop\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -999,7 +1003,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"hazard_max", "int", NULL, "min=15", NULL, 0}, {"history_store", "category", NULL, NULL, confchk_wiredtiger_open_history_store_subconfigs, 1}, {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, - {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, + {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 10}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, {"mmap", "boolean", NULL, NULL, NULL, 0}, {"mmap_all", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, @@ -1021,6 +1025,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," + "\"checkpoint_stop\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -1074,7 +1079,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"hazard_max", "int", NULL, "min=15", NULL, 0}, {"history_store", "category", NULL, NULL, confchk_wiredtiger_open_history_store_subconfigs, 1}, {"io_capacity", "category", NULL, NULL, confchk_wiredtiger_open_io_capacity_subconfigs, 1}, - {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 9}, + {"log", "category", NULL, NULL, confchk_wiredtiger_open_log_subconfigs, 10}, {"lsm_manager", "category", NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2}, {"mmap", "boolean", NULL, NULL, NULL, 0}, {"mmap_all", "boolean", NULL, NULL, NULL, 0}, {"multiprocess", "boolean", NULL, NULL, NULL, 0}, @@ -1096,6 +1101,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," + "\"checkpoint_stop\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -1421,16 +1427,17 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "close_scan_interval=10),hash=(buckets=512,dhandle_buckets=512)," "hazard_max=1000,history_store=(file_max=0),in_memory=false," "io_capacity=(total=0),log=(archive=true,compressor=," - "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," - "mmap_all=false,multiprocess=false,operation_timeout_ms=0," - "operation_tracking=(enabled=false,path=\".\"),readonly=false," - "salvage=false,session_max=100,session_scratch_max=2MB," - "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," + "enabled=false,file_max=100MB,force_write_wait=0," + "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "mmap=true,mmap_all=false,multiprocess=false," + "operation_timeout_ms=0,operation_tracking=(enabled=false," + "path=\".\"),readonly=false,salvage=false,session_max=100," + "session_scratch_max=2MB,session_table_cache=true," + "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=false,on_close=false," + "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," + "tiered_manager=(threads_max=8,threads_min=1,wait=0)," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," "cache_directory=,local_retention=300,name=)," "timing_stress_for_test=,transaction_sync=(enabled=false," @@ -1456,16 +1463,17 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "close_scan_interval=10),hash=(buckets=512,dhandle_buckets=512)," "hazard_max=1000,history_store=(file_max=0),in_memory=false," "io_capacity=(total=0),log=(archive=true,compressor=," - "enabled=false,file_max=100MB,os_cache_dirty_pct=0,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),mmap=true," - "mmap_all=false,multiprocess=false,operation_timeout_ms=0," - "operation_tracking=(enabled=false,path=\".\"),readonly=false," - "salvage=false,session_max=100,session_scratch_max=2MB," - "session_table_cache=true,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),tiered_manager=(threads_max=8,threads_min=1,wait=0)," + "enabled=false,file_max=100MB,force_write_wait=0," + "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "mmap=true,mmap_all=false,multiprocess=false," + "operation_timeout_ms=0,operation_tracking=(enabled=false," + "path=\".\"),readonly=false,salvage=false,session_max=100," + "session_scratch_max=2MB,session_table_cache=true," + "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=false,on_close=false," + "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," + "tiered_manager=(threads_max=8,threads_min=1,wait=0)," "tiered_storage=(auth_token=,bucket=,bucket_prefix=," "cache_directory=,local_retention=300,name=)," "timing_stress_for_test=,transaction_sync=(enabled=false," @@ -1492,10 +1500,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "close_scan_interval=10),hash=(buckets=512,dhandle_buckets=512)," "hazard_max=1000,history_store=(file_max=0),io_capacity=(total=0)" ",log=(archive=true,compressor=,enabled=false,file_max=100MB," - "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," - "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," - "mmap=true,mmap_all=false,multiprocess=false," - "operation_timeout_ms=0,operation_tracking=(enabled=false," + "force_write_wait=0,os_cache_dirty_pct=0,path=\".\",prealloc=true" + ",recover=on,zero_fill=false),lsm_manager=(merge=true," + "worker_thread_max=4),mmap=true,mmap_all=false,multiprocess=false" + ",operation_timeout_ms=0,operation_tracking=(enabled=false," "path=\".\"),readonly=false,salvage=false,session_max=100," "session_scratch_max=2MB,session_table_cache=true," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," @@ -1527,10 +1535,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "close_scan_interval=10),hash=(buckets=512,dhandle_buckets=512)," "hazard_max=1000,history_store=(file_max=0),io_capacity=(total=0)" ",log=(archive=true,compressor=,enabled=false,file_max=100MB," - "os_cache_dirty_pct=0,path=\".\",prealloc=true,recover=on," - "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," - "mmap=true,mmap_all=false,multiprocess=false," - "operation_timeout_ms=0,operation_tracking=(enabled=false," + "force_write_wait=0,os_cache_dirty_pct=0,path=\".\",prealloc=true" + ",recover=on,zero_fill=false),lsm_manager=(merge=true," + "worker_thread_max=4),mmap=true,mmap_all=false,multiprocess=false" + ",operation_timeout_ms=0,operation_tracking=(enabled=false," "path=\".\"),readonly=false,salvage=false,session_max=100," "session_scratch_max=2MB,session_table_cache=true," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index ee79a3602c7..322feebbdcd 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2137,6 +2137,7 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) {"backup_rename", WT_TIMING_STRESS_BACKUP_RENAME}, {"checkpoint_reserved_txnid_delay", WT_TIMING_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY}, {"checkpoint_slow", WT_TIMING_STRESS_CHECKPOINT_SLOW}, + {"checkpoint_stop", WT_TIMING_STRESS_CHECKPOINT_STOP}, {"failpoint_history_delete_key_from_ts", WT_TIMING_STRESS_FAILPOINT_HISTORY_STORE_DELETE_KEY_FROM_TS}, {"history_store_checkpoint_delay", WT_TIMING_STRESS_HS_CHECKPOINT_DELAY}, diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index e4b3c5e981d..85afeb68abf 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -305,6 +305,10 @@ __wt_logmgr_config(WT_SESSION_IMPL *session, const char **cfg, bool reconfig) if (cval.val != 0) conn->log_prealloc = 1; + WT_RET(__wt_config_gets(session, cfg, "log.force_write_wait", &cval)); + if (cval.val != 0) + conn->log_force_write_wait = (uint32_t)cval.val; + /* * Note it's meaningless to reconfigure this value during runtime, it only matters on create * before recovery runs. @@ -832,20 +836,22 @@ __log_server(void *arg) WT_DECL_RET; WT_LOG *log; WT_SESSION_IMPL *session; + uint64_t force_write_time_start, force_write_timediff; uint64_t time_start, time_stop, timediff; bool did_work, signalled; session = arg; conn = S2C(session); log = conn->log; + force_write_timediff = 0; signalled = false; /* - * Set this to the number of milliseconds we want to run archive and pre-allocation. Start it so - * that we run on the first time through. + * Set this to the number of milliseconds we want to run log force write, remove and + * pre-allocation. Start it so that we run on the first time through. */ timediff = WT_THOUSAND; - time_start = __wt_clock(session); + force_write_time_start = time_start = __wt_clock(session); /* * The log server thread does a variety of work. It forces out any buffered log writes. It @@ -863,8 +869,11 @@ __log_server(void *arg) * buffer may need to wait for the write_lsn to advance in the case of a synchronous buffer. * We end up with a hang. */ - WT_ERR_ERROR_OK(__wt_log_force_write(session, 0, &did_work), EBUSY, false); - + if (conn->log_force_write_wait == 0 || + force_write_timediff >= conn->log_force_write_wait * WT_THOUSAND) { + WT_ERR_ERROR_OK(__wt_log_force_write(session, 0, &did_work), EBUSY, false); + force_write_time_start = __wt_clock(session); + } /* * We don't want to archive or pre-allocate files as often as we want to force out log * buffers. Only do it once per second or if the condition was signalled. @@ -902,6 +911,7 @@ __log_server(void *arg) __wt_cond_auto_wait_signal(session, conn->log_cond, did_work, NULL, &signalled); time_stop = __wt_clock(session); timediff = WT_CLOCKDIFF_MS(time_stop, time_start); + force_write_timediff = WT_CLOCKDIFF_MS(time_stop, force_write_time_start); } if (0) { diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 5c91bff7faa..73eee425d13 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -475,6 +475,7 @@ struct __wt_connection_impl { uint32_t log_cursors; /* Log cursor count */ wt_off_t log_dirty_max; /* Log dirty system cache max size */ wt_off_t log_file_max; /* Log file max size */ + uint32_t log_force_write_wait; /* Log force write wait configuration */ const char *log_path; /* Logging path format */ uint32_t log_prealloc; /* Log file pre-allocation */ uint16_t log_req_max; /* Max required log version */ @@ -603,22 +604,23 @@ struct __wt_connection_impl { * Variable with flags for which subsystems the diagnostic stress timing delays have been requested. */ /* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_TIMING_STRESS_AGGRESSIVE_SWEEP 0x0001u -#define WT_TIMING_STRESS_BACKUP_RENAME 0x0002u -#define WT_TIMING_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY 0x0004u -#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x0008u -#define WT_TIMING_STRESS_FAILPOINT_HISTORY_STORE_DELETE_KEY_FROM_TS 0x0010u -#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x0020u -#define WT_TIMING_STRESS_HS_SEARCH 0x0040u -#define WT_TIMING_STRESS_HS_SWEEP 0x0080u -#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x0100u -#define WT_TIMING_STRESS_SPLIT_1 0x0200u -#define WT_TIMING_STRESS_SPLIT_2 0x0400u -#define WT_TIMING_STRESS_SPLIT_3 0x0800u -#define WT_TIMING_STRESS_SPLIT_4 0x1000u -#define WT_TIMING_STRESS_SPLIT_5 0x2000u -#define WT_TIMING_STRESS_SPLIT_6 0x4000u -#define WT_TIMING_STRESS_SPLIT_7 0x8000u +#define WT_TIMING_STRESS_AGGRESSIVE_SWEEP 0x00001u +#define WT_TIMING_STRESS_BACKUP_RENAME 0x00002u +#define WT_TIMING_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY 0x00004u +#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x00008u +#define WT_TIMING_STRESS_CHECKPOINT_STOP 0x00010u +#define WT_TIMING_STRESS_FAILPOINT_HISTORY_STORE_DELETE_KEY_FROM_TS 0x00020u +#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x00040u +#define WT_TIMING_STRESS_HS_SEARCH 0x00080u +#define WT_TIMING_STRESS_HS_SWEEP 0x00100u +#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x00200u +#define WT_TIMING_STRESS_SPLIT_1 0x00400u +#define WT_TIMING_STRESS_SPLIT_2 0x00800u +#define WT_TIMING_STRESS_SPLIT_3 0x01000u +#define WT_TIMING_STRESS_SPLIT_4 0x02000u +#define WT_TIMING_STRESS_SPLIT_5 0x04000u +#define WT_TIMING_STRESS_SPLIT_6 0x08000u +#define WT_TIMING_STRESS_SPLIT_7 0x10000u /* AUTOMATIC FLAG VALUE GENERATION STOP 64 */ uint64_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index a5c1a575853..4d5d0aab440 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -790,6 +790,7 @@ struct __wt_connection_stats { int64_t txn_checkpoint_prep_total; int64_t txn_checkpoint_scrub_target; int64_t txn_checkpoint_scrub_time; + int64_t txn_checkpoint_stop_stress_active; int64_t txn_checkpoint_time_total; int64_t txn_checkpoint; int64_t txn_checkpoint_obsolete_applied; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 0eb9d1bf2bb..592620d0516 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2923,20 +2923,26 @@ struct __wt_connection { * subsystem., a boolean flag; default \c false.} * @config{ file_max, the * maximum size of log files., an integer between 100KB and 2GB; default \c 100MB.} - * @config{ os_cache_dirty_pct, maximum dirty system buffer cache usage\, as - * a percentage of the log's \c file_max. If non-zero\, schedule writes for dirty blocks belonging - * to the log in the system buffer cache after that percentage of the log has been written into the - * buffer cache without an intervening file sync., an integer between 0 and 100; default \c 0.} - * @config{ path, the name of a directory into which log files are written. - * The directory must already exist. If the value is not an absolute path\, the path is relative to - * the database home (see @ref absolute_path for more information)., a string; default \c ".".} - * @config{ prealloc, pre-allocate log files., a boolean flag; default \c - * true.} - * @config{ recover, run recovery or error if recovery needs to run - * after an unclean shutdown., a string\, chosen from the following options: \c "error"\, \c "on"; - * default \c on.} - * @config{ zero_fill, manually write zeroes into log files., - * a boolean flag; default \c false.} + * @config{ force_write_wait, enable code that interrupts the usual timing of + * flushing the log from the internal log server thread with a goal of uncovering race conditions. + * This option is intended for use with internal stress testing of WiredTiger., an integer between 1 + * and 60; default \c 0.} + * @config{ os_cache_dirty_pct, maximum dirty system + * buffer cache usage\, as a percentage of the log's \c file_max. If non-zero\, schedule writes for + * dirty blocks belonging to the log in the system buffer cache after that percentage of the log has + * been written into the buffer cache without an intervening file sync., an integer between 0 and + * 100; default \c 0.} + * @config{ path, the name of a directory into which log + * files are written. The directory must already exist. If the value is not an absolute path\, the + * path is relative to the database home (see @ref absolute_path for more information)., a string; + * default \c ".".} + * @config{ prealloc, pre-allocate log files., a boolean + * flag; default \c true.} + * @config{ recover, run recovery or error if + * recovery needs to run after an unclean shutdown., a string\, chosen from the following options: + * \c "error"\, \c "on"; default \c on.} + * @config{ zero_fill, manually write + * zeroes into log files., a boolean flag; default \c false.} * @config{ ),,} * @config{lsm_manager = (, configure database wide options for LSM tree management. The LSM * manager is started automatically the first time an LSM tree is opened. The LSM manager uses a @@ -6178,59 +6184,61 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1475 /*! transaction: transaction checkpoint scrub time (msecs) */ #define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1476 +/*! transaction: transaction checkpoint stop timing stress active */ +#define WT_STAT_CONN_TXN_CHECKPOINT_STOP_STRESS_ACTIVE 1477 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1477 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1478 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1478 +#define WT_STAT_CONN_TXN_CHECKPOINT 1479 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1479 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1480 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1480 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1481 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1481 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1482 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1482 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1483 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1483 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1484 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1484 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1485 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1485 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1486 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1486 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1487 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1487 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1488 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1488 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1489 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1489 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1490 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1490 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1491 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1491 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1492 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1492 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1493 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1493 +#define WT_STAT_CONN_TXN_COMMIT 1494 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1494 +#define WT_STAT_CONN_TXN_ROLLBACK 1495 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1495 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1496 /*! * @} diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 1bffa501958..6e302416d70 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1520,6 +1520,7 @@ static const char *const __stats_connection_desc[] = { "transaction: transaction checkpoint prepare total time (msecs)", "transaction: transaction checkpoint scrub dirty target", "transaction: transaction checkpoint scrub time (msecs)", + "transaction: transaction checkpoint stop timing stress active", "transaction: transaction checkpoint total time (msecs)", "transaction: transaction checkpoints", "transaction: transaction checkpoints due to obsolete pages", @@ -2057,6 +2058,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing txn_checkpoint_prep_total */ /* not clearing txn_checkpoint_scrub_target */ /* not clearing txn_checkpoint_scrub_time */ + /* not clearing txn_checkpoint_stop_stress_active */ /* not clearing txn_checkpoint_time_total */ stats->txn_checkpoint = 0; stats->txn_checkpoint_obsolete_applied = 0; @@ -2615,6 +2617,7 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->txn_checkpoint_prep_total += WT_STAT_READ(from, txn_checkpoint_prep_total); to->txn_checkpoint_scrub_target += WT_STAT_READ(from, txn_checkpoint_scrub_target); to->txn_checkpoint_scrub_time += WT_STAT_READ(from, txn_checkpoint_scrub_time); + to->txn_checkpoint_stop_stress_active += WT_STAT_READ(from, txn_checkpoint_stop_stress_active); to->txn_checkpoint_time_total += WT_STAT_READ(from, txn_checkpoint_time_total); to->txn_checkpoint += WT_STAT_READ(from, txn_checkpoint); to->txn_checkpoint_obsolete_applied += WT_STAT_READ(from, txn_checkpoint_obsolete_applied); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 86ba9274086..e4ab6778af8 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1021,6 +1021,15 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_txn_commit(session, NULL)); /* + * Flush all the logs that are generated during the checkpoint. It is possible that checkpoint + * may include the changes that are written in parallel by an eviction. To have a consistent + * view of the data, make sure that all the logs are flushed to disk before the checkpoint is + * complete. + */ + if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) + WT_ERR(__wt_log_flush(session, WT_LOG_FSYNC)); + + /* * Ensure that the metadata changes are durable before the checkpoint is resolved. Do this by * either checkpointing the metadata or syncing the log file. Recovery relies on the checkpoint * LSN in the metadata only being updated by full checkpoints so only checkpoint the metadata @@ -1047,6 +1056,11 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_WITH_DHANDLE(session, WT_SESSION_META_DHANDLE(session), ret = __wt_txn_checkpoint_log(session, false, WT_TXN_LOG_CKPT_SYNC, NULL)); + WT_STAT_CONN_SET(session, txn_checkpoint_stop_stress_active, 1); + /* Wait prior to flush the checkpoint stop log record. */ + __checkpoint_timing_stress(session, WT_TIMING_STRESS_CHECKPOINT_STOP, &tsp); + WT_STAT_CONN_SET(session, txn_checkpoint_stop_stress_active, 0); + /* * Now that the metadata is stable, re-open the metadata file for regular eviction by clearing * the checkpoint_pinned flag. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable35.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable35.py new file mode 100755 index 00000000000..218a4330251 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable35.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest, threading, time +from wtdataset import SimpleDataSet +from wtthread import checkpoint_thread +from helper import simulate_crash_restart +from wiredtiger import stat +from helper import copy_wiredtiger_home +from wtscenario import make_scenarios +from test_rollback_to_stable01 import test_rollback_to_stable_base + +# test_rollback_to_stable35.py +# Test that log is flushed for all writes that occurred in the checkpoint. +class test_rollback_to_stable35(test_rollback_to_stable_base): + + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + scenarios = make_scenarios(format_values) + + def large_updates(self, uri_1, uri_2, value, ds_1, ds_2, nrows): + # Update a large number of records. + session = self.session + cursor_1 = session.open_cursor(uri_1) + cursor_2 = session.open_cursor(uri_2) + for i in range(1, nrows + 1): + session.begin_transaction() + cursor_1[ds_1.key(i)] = value + cursor_2[ds_2.key(i)] = value + session.commit_transaction() + cursor_1.close() + cursor_2.close() + + def check(self, check_value, uri_1, uri_2, nrows): + session = self.session + session.begin_transaction() + cursor_1 = session.open_cursor(uri_1) + cursor_2 = session.open_cursor(uri_2) + count = 0 + for k, v in cursor_1: + self.assertEqual(v, check_value) + count += 1 + self.assertEqual(count, nrows) + count = 0 + for k, v in cursor_2: + self.assertEqual(v, check_value) + count += 1 + self.assertEqual(count, nrows) + session.commit_transaction() + cursor_1.close() + cursor_2.close() + + def conn_config(self): + config = 'cache_size=50MB,statistics=(all),log=(enabled,force_write_wait=60),timing_stress_for_test=[checkpoint_slow, checkpoint_stop]' + return config + + def test_rollback_to_stable(self): + nrows = 10 + + # Create two tables. + uri_1 = "table:rollback_to_stable35_1" + ds_1 = SimpleDataSet( + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format) + ds_1.populate() + + uri_2 = "table:rollback_to_stable35_2" + ds_2 = SimpleDataSet( + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format) + ds_2.populate() + + if self.value_format == '8t': + valuea = 97 + valueb = 98 + valuec = 99 + else: + valuea = "aaaaa" * 100 + valueb = "bbbbb" * 100 + valuec = "ccccc" * 100 + + self.large_updates(uri_1, uri_2, valuea, ds_1, ds_2, nrows) + self.check(valuea, uri_1, uri_2, nrows) + + # Start a long running transaction and keep it open. + session_2 = self.conn.open_session() + session_2.begin_transaction() + + self.large_updates(uri_1, uri_2, valueb, ds_1, ds_2, nrows) + self.check(valueb, uri_1, uri_2, nrows) + + # Create a checkpoint thread + done = threading.Event() + ckpt = checkpoint_thread(self.conn, done) + try: + ckpt.start() + # Wait for checkpoint to start before committing. + ckpt_started = 0 + while not ckpt_started: + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] + stat_cursor.close() + time.sleep(1) + + self.large_updates(uri_1, uri_2, valuec, ds_1, ds_2, nrows) + self.check(valuec, uri_1, uri_2, nrows) + + # Evict the data. + self.evict_cursor(uri_1, nrows, valuec) + + # Wait for checkpoint stop timing stress to copy the database. + ckpt_stop_timing_stress = 0 + while not ckpt_stop_timing_stress: + time.sleep(1) + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_stop_timing_stress = stat_cursor[stat.conn.txn_checkpoint_stop_stress_active][2] + stat_cursor.close() + + copy_wiredtiger_home(self, '.', "RESTART") + + finally: + done.set() + ckpt.join() + self.session.checkpoint() + + # Clear all running transactions before rollback to stable. + session_2.commit_transaction() + session_2.close() + + # Open the new directory + self.close_conn() + self.conn_config = 'cache_size=50MB,statistics=(all),log=(enabled)' + conn = self.setUpConnectionOpen("RESTART") + self.session = self.setUpSessionOpen(conn) + + self.check(valuec, uri_1, uri_2, nrows) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + calls = stat_cursor[stat.conn.txn_rts][2] + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] + keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] + pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] + stat_cursor.close() + + self.assertEqual(calls, 0) + self.assertEqual(keys_removed, 0) + self.assertEqual(keys_restored, 0) + self.assertEqual(pages_visited, 0) + self.assertEqual(upd_aborted, 0) + self.assertGreaterEqual(hs_removed, 0) + +if __name__ == '__main__': + wttest.run() |