diff options
author | Luke Chen <luke.chen@mongodb.com> | 2022-12-05 16:26:02 +1100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-12-05 05:54:13 +0000 |
commit | 5832ea886e75c568b24d9bfb763571757d06e22f (patch) | |
tree | ab2358cfdb3cadb754a440a17c34b721d7baea01 | |
parent | 55da6eb8d2ee810cba946c4bf2606c15afd1f275 (diff) | |
download | mongo-5832ea886e75c568b24d9bfb763571757d06e22f.tar.gz |
Import wiredtiger: b839ed28a19d5316ac56ea97ab2efe8f607a94d2 from branch mongodb-5.0
ref: 2f5d15ff28..b839ed28a1
for: 5.0.15
WT-9592 Fix rollback to stable not clearing the WT_UPDATE_TO_DELETE_FROM_HS flag
17 files changed, 389 insertions, 72 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index f3fe230611e..8f9dacf4aeb 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -848,7 +848,8 @@ connection_runtime_config = [ type='list', undoc=True, choices=[ 'aggressive_sweep', 'backup_rename', 'checkpoint_reserved_txnid_delay', 'checkpoint_slow', - 'checkpoint_stop', 'failpoint_history_store_delete_key_from_ts', + 'checkpoint_stop', 'failpoint_eviction_fail_after_reconciliation', + 'failpoint_history_store_delete_key_from_ts', 'history_store_checkpoint_delay', 'history_store_search', 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', 'split_2', 'split_3', 'split_4', 'split_5', 'split_6', 'split_7']), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 63bb920d1ae..62c03aa0371 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "2f5d15ff28b6fd3351524cc8bcbd5b6ecc489670" + "commit": "b839ed28a19d5316ac56ea97ab2efe8f607a94d2" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index f2efc74faf6..272ff43f42a 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -153,6 +153,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," "\"checkpoint_stop\"," + "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -884,6 +885,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," "\"checkpoint_stop\"," + "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -966,6 +968,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," "\"checkpoint_stop\"," + "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -1045,6 +1048,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," "\"checkpoint_stop\"," + "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," @@ -1122,6 +1126,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_reserved_txnid_delay\",\"checkpoint_slow\"," "\"checkpoint_stop\"," + "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 0c5520edb50..47d7ed6f66e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2190,6 +2190,8 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) {"checkpoint_reserved_txnid_delay", WT_TIMING_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY}, {"checkpoint_slow", WT_TIMING_STRESS_CHECKPOINT_SLOW}, {"checkpoint_stop", WT_TIMING_STRESS_CHECKPOINT_STOP}, + {"failpoint_eviction_fail_after_reconciliation", + WT_TIMING_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION}, {"failpoint_history_delete_key_from_ts", WT_TIMING_STRESS_FAILPOINT_HISTORY_STORE_DELETE_KEY_FROM_TS}, {"history_store_checkpoint_delay", WT_TIMING_STRESS_HS_CHECKPOINT_DELAY}, @@ -2212,9 +2214,8 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) flags = 0; for (ft = stress_types; ft->name != NULL; ft++) { - if ((ret = __wt_config_subgets(session, &cval, ft->name, &sval)) == 0 && sval.val != 0) { + if ((ret = __wt_config_subgets(session, &cval, ft->name, &sval)) == 0 && sval.val != 0) LF_SET(ft->flag); - } WT_RET_NOTFOUND_OK(ret); } diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index de249ec4a4b..62c8c53ea4f 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -761,5 +761,9 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t evict_flags, bool !__wt_page_is_modified(page) || LF_ISSET(WT_REC_HS | WT_REC_IN_MEMORY) || WT_IS_METADATA(btree->dhandle)); + /* Fail 0.1% of the time. */ + if (__wt_failpoint(session, WT_TIMING_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION, 10)) + return (EBUSY); + return (0); } diff --git a/src/third_party/wiredtiger/src/history/hs_rec.c b/src/third_party/wiredtiger/src/history/hs_rec.c index 1b8295de491..01d3d40a7c1 100644 --- a/src/third_party/wiredtiger/src/history/hs_rec.c +++ b/src/third_party/wiredtiger/src/history/hs_rec.c @@ -467,6 +467,11 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_MULTI *mult continue; if (F_ISSET(upd, WT_UPDATE_TO_DELETE_FROM_HS)) { + /* + * If we want to remove an update from the history store in WiredTiger, it must be + * in history store. + */ + WT_ASSERT(session, F_ISSET(upd, WT_UPDATE_HS | WT_UPDATE_RESTORED_FROM_HS)); if (upd->type == WT_UPDATE_TOMBSTONE) delete_tombstone = upd; else { diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 0919d91b0fb..e1f78300e68 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -572,18 +572,19 @@ struct __wt_connection_impl { #define WT_TIMING_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY 0x00004u #define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x00008u #define WT_TIMING_STRESS_CHECKPOINT_STOP 0x00010u -#define WT_TIMING_STRESS_FAILPOINT_HISTORY_STORE_DELETE_KEY_FROM_TS 0x00020u -#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x00040u -#define WT_TIMING_STRESS_HS_SEARCH 0x00080u -#define WT_TIMING_STRESS_HS_SWEEP 0x00100u -#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x00200u -#define WT_TIMING_STRESS_SPLIT_1 0x00400u -#define WT_TIMING_STRESS_SPLIT_2 0x00800u -#define WT_TIMING_STRESS_SPLIT_3 0x01000u -#define WT_TIMING_STRESS_SPLIT_4 0x02000u -#define WT_TIMING_STRESS_SPLIT_5 0x04000u -#define WT_TIMING_STRESS_SPLIT_6 0x08000u -#define WT_TIMING_STRESS_SPLIT_7 0x10000u +#define WT_TIMING_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION 0x00020u +#define WT_TIMING_STRESS_FAILPOINT_HISTORY_STORE_DELETE_KEY_FROM_TS 0x00040u +#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x00080u +#define WT_TIMING_STRESS_HS_SEARCH 0x00100u +#define WT_TIMING_STRESS_HS_SWEEP 0x00200u +#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x00400u +#define WT_TIMING_STRESS_SPLIT_1 0x00800u +#define WT_TIMING_STRESS_SPLIT_2 0x01000u +#define WT_TIMING_STRESS_SPLIT_3 0x02000u +#define WT_TIMING_STRESS_SPLIT_4 0x04000u +#define WT_TIMING_STRESS_SPLIT_5 0x08000u +#define WT_TIMING_STRESS_SPLIT_6 0x10000u +#define WT_TIMING_STRESS_SPLIT_7 0x20000u /* AUTOMATIC FLAG VALUE GENERATION STOP 64 */ uint64_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index c5e73637d30..817fdc56f9b 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -120,13 +120,12 @@ __rollback_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *first } /* - * Clear the history store flag for the stable update to indicate that this update should not be - * written into the history store later, when all the aborted updates are removed from the - * history store. The next time when this update is moved into the history store, it will have a - * different stop time point. + * Clear the history store flags for the stable update to indicate that this update should be + * written to the history store later. The next time when this update is moved into the history + * store, it will have a different stop time point. */ if (stable_upd != NULL) { - if (F_ISSET(stable_upd, WT_UPDATE_HS)) { + if (F_ISSET(stable_upd, WT_UPDATE_HS | WT_UPDATE_TO_DELETE_FROM_HS)) { /* Find the update following a stable tombstone. */ if (stable_upd->type == WT_UPDATE_TOMBSTONE) { tombstone = stable_upd; @@ -135,7 +134,7 @@ __rollback_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *first if (stable_upd->txnid != WT_TXN_ABORTED) { WT_ASSERT(session, stable_upd->type != WT_UPDATE_TOMBSTONE && - F_ISSET(stable_upd, WT_UPDATE_HS)); + F_ISSET(stable_upd, WT_UPDATE_HS | WT_UPDATE_TO_DELETE_FROM_HS)); break; } } @@ -151,13 +150,13 @@ __rollback_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *first session, key, stable_upd == NULL ? tombstone->start_ts : stable_upd->start_ts)); /* - * Clear the history store flag for the first stable update. Otherwise, it will not be + * Clear the history store flags for the first stable update. Otherwise, it will not be * moved to history store again. */ if (stable_upd != NULL) - F_CLR(stable_upd, WT_UPDATE_HS); + F_CLR(stable_upd, WT_UPDATE_HS | WT_UPDATE_TO_DELETE_FROM_HS); if (tombstone != NULL) - F_CLR(tombstone, WT_UPDATE_HS); + F_CLR(tombstone, WT_UPDATE_HS | WT_UPDATE_TO_DELETE_FROM_HS); } if (stable_update_found != NULL) *stable_update_found = true; diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 2c3ab04c7f0..4c95b094b9c 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -64,6 +64,7 @@ main(int argc, char *argv[]) g.ntables = 3; g.nworkers = 1; g.sweep_stress = g.use_timestamps = false; + g.failpoint_eviction_fail_after_reconciliation = false; g.failpoint_hs_delete_key_from_ts = false; g.hs_checkpoint_timing_stress = g.reserved_txnid_timing_stress = false; g.checkpoint_slow_timing_stress = false; @@ -123,6 +124,9 @@ main(int argc, char *argv[]) case '5': g.checkpoint_slow_timing_stress = true; break; + case '7': + g.failpoint_eviction_fail_after_reconciliation = true; + break; default: return (usage()); } @@ -266,7 +270,10 @@ wt_connect(const char *config_open) g.checkpoint_slow_timing_stress) { timing_stress = true; testutil_check(__wt_snprintf(timing_stress_cofing, sizeof(timing_stress_cofing), - ",timing_stress_for_test=[%s%s%s%s%s]", g.sweep_stress ? "aggressive_sweep" : "", + ",timing_stress_for_test=[%s%s%s%s%s%s]", g.sweep_stress ? "aggressive_sweep" : "", + g.failpoint_eviction_fail_after_reconciliation ? + "failpoint_eviction_fail_after_reconciliation" : + "", g.failpoint_hs_delete_key_from_ts ? "failpoint_history_store_delete_key_from_ts" : "", g.hs_checkpoint_timing_stress ? "history_store_checkpoint_delay" : "", g.reserved_txnid_timing_stress ? "checkpoint_reserved_txnid_delay" : "", @@ -430,6 +437,8 @@ usage(void) "\t\t3: hs_checkpoint_timing_stress\n" "\t\t4: reserved_txnid_timing_stress\n" "\t\t5: checkpoint_slow_timing_stress\n" + "\t\t6: evict_reposition_timing_stress\n" + "\t\t7: failpoint_eviction_fail_after_reconciliation\n" "\t-T specify a table configuration\n" "\t-t set a file type ( col | mix | row | lsm )\n" "\t-v verify only\n" diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index 93f76ec5e1c..b9ec8d47c8d 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -52,35 +52,37 @@ typedef struct { } COOKIE; typedef struct { - char *home; /* Home directory */ - const char *checkpoint_name; /* Checkpoint name */ - WT_CONNECTION *conn; /* WiredTiger connection */ - bool debug_mode; /* History store stress test */ - u_int nkeys; /* Keys to load */ - u_int nops; /* Operations per thread */ - FILE *logfp; /* Message log file. */ - int nworkers; /* Number workers configured */ - int ntables; /* Number tables configured */ - int ntables_created; /* Number tables opened */ - volatile int running; /* Whether to stop */ - int status; /* Exit status */ - bool sweep_stress; /* Sweep stress test */ - bool failpoint_hs_delete_key_from_ts; /* Failpoint for hs key deletion. */ - bool failpoint_hs_insert_1; /* Failpoint for hs insertion. */ - bool failpoint_hs_insert_2; /* Failpoint for hs insertion. */ - bool hs_checkpoint_timing_stress; /* History store checkpoint timing stress */ - bool reserved_txnid_timing_stress; /* Reserved transaction id timing stress */ - bool checkpoint_slow_timing_stress; /* Checkpoint slow timing stress */ - uint64_t ts_oldest; /* Current oldest timestamp */ - uint64_t ts_stable; /* Current stable timestamp */ - bool mixed_mode_deletes; /* Run with mixed mode deletes */ - bool use_timestamps; /* Use txn timestamps */ - bool race_timetamps; /* Async update to oldest timestamp */ - bool prepare; /* Use prepare transactions */ - COOKIE *cookies; /* Per-thread info */ - WT_RWLOCK clock_lock; /* Clock synchronization */ - wt_thread_t checkpoint_thread; /* Checkpoint thread */ - wt_thread_t clock_thread; /* Clock thread */ + char *home; /* Home directory */ + const char *checkpoint_name; /* Checkpoint name */ + WT_CONNECTION *conn; /* WiredTiger connection */ + bool debug_mode; /* History store stress test */ + u_int nkeys; /* Keys to load */ + u_int nops; /* Operations per thread */ + FILE *logfp; /* Message log file. */ + int nworkers; /* Number workers configured */ + int ntables; /* Number tables configured */ + int ntables_created; /* Number tables opened */ + volatile int running; /* Whether to stop */ + int status; /* Exit status */ + bool sweep_stress; /* Sweep stress test */ + bool failpoint_hs_delete_key_from_ts; /* Failpoint for hs key deletion. */ + bool failpoint_hs_insert_1; /* Failpoint for hs insertion. */ + bool failpoint_hs_insert_2; /* Failpoint for hs insertion. */ + bool hs_checkpoint_timing_stress; /* History store checkpoint timing stress */ + bool reserved_txnid_timing_stress; /* Reserved transaction id timing stress */ + bool checkpoint_slow_timing_stress; /* Checkpoint slow timing stress */ + uint64_t ts_oldest; /* Current oldest timestamp */ + uint64_t ts_stable; /* Current stable timestamp */ + bool failpoint_eviction_fail_after_reconciliation; /*Fail point for eviction fail after + reconciliation. */ + bool mixed_mode_deletes; /* Run with mixed mode deletes */ + bool use_timestamps; /* Use txn timestamps */ + bool race_timetamps; /* Async update to oldest timestamp */ + bool prepare; /* Use prepare transactions */ + COOKIE *cookies; /* Per-thread info */ + WT_RWLOCK clock_lock; /* Clock synchronization */ + wt_thread_t checkpoint_thread; /* Checkpoint thread */ + wt_thread_t clock_thread; /* Clock thread */ } GLOBAL; extern GLOBAL g; diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 831d99579fc..079bd544565 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -2697,6 +2697,20 @@ tasks: set -o errexit set -o verbose ${test_env_vars|} ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 5" WT_TEST.$t' 120 + + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s7 + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/cmake_build/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + ${test_env_vars|} ../../../tools/run_parallel.sh 'nice ../../../test/checkpoint/recovery-test.sh "${data_validation_stress_test_args} -s 7" WT_TEST.$t' 120 - name: format-failure-configs-test depends_on: @@ -3671,6 +3685,7 @@ buildvariants: - name: data-validation-stress-test-checkpoint-fp-hs-insert-s3 - name: data-validation-stress-test-checkpoint-fp-hs-insert-s4 - name: data-validation-stress-test-checkpoint-fp-hs-insert-s5 + - name: data-validation-stress-test-checkpoint-fp-hs-insert-s7 - name: ubuntu2004-cmake display_name: "* Ubuntu 20.04 CMake" diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 314dca29b34..04a9e20819f 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -115,21 +115,22 @@ typedef struct { #define V_GLOBAL_STRESS_CHECKPOINT 83 #define V_GLOBAL_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY 84 #define V_GLOBAL_STRESS_CHECKPOINT_PREPARE 85 -#define V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS 86 -#define V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY 87 -#define V_GLOBAL_STRESS_HS_SEARCH 88 -#define V_GLOBAL_STRESS_HS_SWEEP 89 -#define V_GLOBAL_STRESS_SPLIT_1 90 -#define V_GLOBAL_STRESS_SPLIT_2 91 -#define V_GLOBAL_STRESS_SPLIT_3 92 -#define V_GLOBAL_STRESS_SPLIT_4 93 -#define V_GLOBAL_STRESS_SPLIT_5 94 -#define V_GLOBAL_STRESS_SPLIT_6 95 -#define V_GLOBAL_STRESS_SPLIT_7 96 -#define V_GLOBAL_TRANSACTION_IMPLICIT 97 -#define V_GLOBAL_TRANSACTION_TIMESTAMPS 98 -#define V_GLOBAL_WIREDTIGER_CONFIG 99 -#define V_GLOBAL_WIREDTIGER_RWLOCK 100 -#define V_GLOBAL_WIREDTIGER_LEAK_MEMORY 101 +#define V_GLOBAL_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION 86 +#define V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS 87 +#define V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY 88 +#define V_GLOBAL_STRESS_HS_SEARCH 89 +#define V_GLOBAL_STRESS_HS_SWEEP 90 +#define V_GLOBAL_STRESS_SPLIT_1 91 +#define V_GLOBAL_STRESS_SPLIT_2 92 +#define V_GLOBAL_STRESS_SPLIT_3 93 +#define V_GLOBAL_STRESS_SPLIT_4 94 +#define V_GLOBAL_STRESS_SPLIT_5 95 +#define V_GLOBAL_STRESS_SPLIT_6 96 +#define V_GLOBAL_STRESS_SPLIT_7 97 +#define V_GLOBAL_TRANSACTION_IMPLICIT 98 +#define V_GLOBAL_TRANSACTION_TIMESTAMPS 99 +#define V_GLOBAL_WIREDTIGER_CONFIG 100 +#define V_GLOBAL_WIREDTIGER_RWLOCK 101 +#define V_GLOBAL_WIREDTIGER_LEAK_MEMORY 102 -#define V_ELEMENT_COUNT 102 +#define V_ELEMENT_COUNT 103 diff --git a/src/third_party/wiredtiger/test/format/config.sh b/src/third_party/wiredtiger/test/format/config.sh index 57d2844653d..1eec7ecd544 100644 --- a/src/third_party/wiredtiger/test/format/config.sh +++ b/src/third_party/wiredtiger/test/format/config.sh @@ -248,6 +248,8 @@ CONFIG configuration_list[] = { {"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0} +{"stress.failpoint_eviction_fail_after_reconciliation", "stress failpoint eviction fail after reconciliation", C_BOOL, 30, 0, 0} + {"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", C_BOOL, 30, 0, 0} {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", C_BOOL, 2, 0, 0} diff --git a/src/third_party/wiredtiger/test/format/config_def.c b/src/third_party/wiredtiger/test/format/config_def.c index 078a49377df..a9442d77a15 100644 --- a/src/third_party/wiredtiger/test/format/config_def.c +++ b/src/third_party/wiredtiger/test/format/config_def.c @@ -266,6 +266,9 @@ CONFIG configuration_list[] = { {"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT_PREPARE}, + {"stress.failpoint_eviction_fail_after_reconciliation", "stress failpoint eviction fail after reconciliation", + C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION}, + {"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS}, diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index a999cc52d4f..d6d2157896b 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -158,6 +158,8 @@ configure_timing_stress(char *p, size_t max) CONFIG_APPEND(p, ",prepare_checkpoint_delay"); if (GV(STRESS_CHECKPOINT_RESERVED_TXNID_DELAY)) CONFIG_APPEND(p, ",checkpoint_reserved_txnid_delay"); + if (GV(STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION)) + CONFIG_APPEND(p, ",failpoint_eviction_fail_after_reconciliation"); if (GV(STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS)) CONFIG_APPEND(p, ",failpoint_history_store_delete_key_from_ts"); if (GV(STRESS_HS_CHECKPOINT_DELAY)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare22.py b/src/third_party/wiredtiger/test/suite/test_prepare22.py new file mode 100644 index 00000000000..e4f4b45e7e2 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare22.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_prepare22.py +# Test prepare with rollback to stable without failed eviction. +class test_prepare22(wttest.WiredTigerTestCase): + + format_values = [ + ('column', dict(key_format='r', value_format='S')), + #('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + delete = [ + ('delete', dict(delete=True)), + ('non-delete', dict(delete=False)), + ] + + scenarios = make_scenarios(format_values, delete) + + def test_prepare22(self): + uri = "table:test_prepare22" + self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "a" + value_b = "b" + value_c = "c" + + # Pin oldest timestamp to 1 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + + # Do the first update + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[1] = value_a + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) + + # Do the second update + self.session.begin_transaction() + cursor[1] = value_b + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) + + if self.delete: + self.session.begin_transaction() + cursor.set_key(1) + cursor.remove() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) + + # Do a prepared update + self.session.begin_transaction() + cursor[1] = value_c + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) + + # Evict the page + session2 = self.conn.open_session() + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(20)) + self.assertEquals(evict_cursor[1], value_b) + evict_cursor.reset() + evict_cursor.close() + session2.rollback_transaction() + + # Ensure the history store is checkpointed + session2.checkpoint() + + # Rollback the prepared transaction + self.session.rollback_transaction() + + # Set stable timestamp to 30 + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) + + # Call rollback to stable + self.conn.rollback_to_stable() + + # Evict the page again + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + session2.begin_transaction('read_timestamp=' + self.timestamp_str(20)) + self.assertEquals(evict_cursor[1], value_b) + evict_cursor.reset() + evict_cursor.close() + session2.rollback_transaction() + + # Verify we can still read back value a + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(10)) + self.assertEquals(cursor[1], value_a) + self.session.rollback_transaction() + + # Verify we can still read back value b + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) + self.assertEquals(cursor[1], value_b) + self.session.rollback_transaction() + + # Verify we can still read back the deletion + if self.delete: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) + if self.value_format == '8t': + self.assertEquals(cursor[1], 0) + else: + cursor.set_key(1) + self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare23.py b/src/third_party/wiredtiger/test/suite/test_prepare23.py new file mode 100644 index 00000000000..927f3aae7c5 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare23.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_prepare23.py +# Test prepare rollback with rollback to stable and failed eviction. +class test_prepare23(wttest.WiredTigerTestCase): + conn_config = 'timing_stress_for_test=[failpoint_eviction_fail_after_reconciliation]' + + format_values = [ + ('column', dict(key_format='r', value_format='S')), + #('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + delete = [ + ('delete', dict(delete=True)), + ('non-delete', dict(delete=False)), + ] + + scenarios = make_scenarios(format_values, delete) + + def test_prepare23(self): + uri = "table:test_prepare23" + self.session.create(uri, 'key_format=' + self.key_format + ',value_format=' + self.value_format) + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "a" + value_b = "b" + value_c = "c" + + # Pin oldest timestamp to 1 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) + + cursor = self.session.open_cursor(uri) + session2 = self.conn.open_session() + evict_cursor = session2.open_cursor(uri, None, 'debug=(release_evict)') + ts = 0 + for i in range (1, 1001): + # Do the first update + self.session.begin_transaction() + cursor[i] = value_a + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 10)) + + # Do the second update + self.session.begin_transaction() + cursor[i] = value_b + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 20)) + + if self.delete: + self.session.begin_transaction() + cursor.set_key(i) + cursor.remove() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts + 30)) + + # Do a prepared update + self.session.begin_transaction() + cursor[i] = value_c + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(ts + 40)) + + # Evict the page + session2.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts + 20)) + self.assertEquals(evict_cursor[i], value_b) + evict_cursor.reset() + session2.rollback_transaction() + + # Rollback the prepared transaction + self.session.rollback_transaction() + + # Set stable timestamp to 30 * i + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(ts + 30)) + + # Call rollback to stable + self.conn.rollback_to_stable() + + # Verify we can still read back value a + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 10)) + self.assertEquals(cursor[i], value_a) + self.session.rollback_transaction() + + # Verify we can still read back value b + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 20)) + self.assertEquals(cursor[i], value_b) + self.session.rollback_transaction() + + # Verify we can still read back the deletion + if self.delete: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts + 30)) + if self.value_format == '8t': + self.assertEquals(cursor[i], 0) + else: + cursor.set_key(i) + self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + + ts += 40 + +if __name__ == '__main__': + wttest.run() |