diff options
author | Luke Chen <luke.chen@mongodb.com> | 2023-02-27 16:23:18 +1100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-02-27 06:06:17 +0000 |
commit | e29864141fe251a370fc1b223242ae846f6c4b17 (patch) | |
tree | 080c7a96392c158f333394164448870616569f82 | |
parent | 48a5a5e48ae87782d33306b143fd7bf397591dde (diff) | |
download | mongo-e29864141fe251a370fc1b223242ae846f6c4b17.tar.gz |
Import wiredtiger: b7f971f04a0864fab42368d594251b3fb1d36477 from branch mongodb-6.3
ref: 772a20a408..b7f971f04a
for: 6.3.0-rc1
WT-10575 Fix checkpoint cursor to return only stable data (6.3 backport)
-rw-r--r-- | src/third_party/wiredtiger/dist/api_data.py | 4 | ||||
-rw-r--r-- | src/third_party/wiredtiger/import.data | 2 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/btree/bt_delete.c | 7 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/config/config_def.c | 25 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/conn/conn_api.c | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/btree_inline.h | 6 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/connection.h | 39 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/extern.h | 4 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/txn.h | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/txn_inline.h | 20 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/lsm/lsm_cursor.c | 3 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/txn/txn.c | 3 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/txn/txn_ckpt.c | 6 | ||||
-rw-r--r-- | src/third_party/wiredtiger/test/suite/test_checkpoint21.py | 4 | ||||
-rw-r--r-- | src/third_party/wiredtiger/test/suite/test_checkpoint28.py | 160 |
15 files changed, 239 insertions, 46 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index b32f4b331c2..0b3722b930c 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -795,8 +795,8 @@ connection_runtime_config = [ stress testing of WiredTiger.''', type='list', undoc=True, choices=[ - 'aggressive_sweep', 'backup_rename', 'checkpoint_evict_page', 'checkpoint_slow', - 'checkpoint_stop', 'compact_slow', 'evict_reposition', + 'aggressive_sweep', 'backup_rename', 'checkpoint_evict_page', 'checkpoint_handle', + 'checkpoint_slow', 'checkpoint_stop', 'compact_slow', 'evict_reposition', 'failpoint_eviction_fail_after_reconciliation', 'failpoint_history_store_delete_key_from_ts', 'history_store_checkpoint_delay', 'history_store_search', 'history_store_sweep_race', 'prepare_checkpoint_delay', diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index da38d282de8..1def05e4b5d 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-6.3", - "commit": "772a20a40858ce26e500de46caf4d278d1cd7998" + "commit": "b7f971f04a0864fab42368d594251b3fb1d36477" } diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index 554792bb898..b094ff58372 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -166,9 +166,14 @@ __wt_delete_page(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) goto err; if (addr.ta.prepare) goto err; - /* History store data are always visible. No need to check visibility. */ + /* + * History store data are always visible. No need to check visibility. Other than history store, + * use the max durable timestamp that is available in the page aggregation for the visibility + * checks as we do not track the aggregated commit timestamp. + */ if (!WT_IS_HS(session->dhandle) && !__wt_txn_visible(session, addr.ta.newest_txn, + WT_MAX(addr.ta.newest_start_durable_ts, addr.ta.newest_stop_durable_ts), WT_MAX(addr.ta.newest_start_durable_ts, addr.ta.newest_stop_durable_ts))) goto err; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index f9b6b3472c6..3e2d04ccb6f 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -159,8 +159,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { confchk_WT_CONNECTION_reconfigure_tiered_storage_subconfigs, 1}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," - "\"checkpoint_evict_page\",\"checkpoint_slow\"," - "\"checkpoint_stop\",\"compact_slow\",\"evict_reposition\"," + "\"checkpoint_evict_page\",\"checkpoint_handle\"," + "\"checkpoint_slow\",\"checkpoint_stop\",\"compact_slow\"," + "\"evict_reposition\"," "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," @@ -920,8 +921,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 8}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," - "\"checkpoint_evict_page\",\"checkpoint_slow\"," - "\"checkpoint_stop\",\"compact_slow\",\"evict_reposition\"," + "\"checkpoint_evict_page\",\"checkpoint_handle\"," + "\"checkpoint_slow\",\"checkpoint_stop\",\"compact_slow\"," + "\"evict_reposition\"," "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," @@ -1010,8 +1012,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 8}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," - "\"checkpoint_evict_page\",\"checkpoint_slow\"," - "\"checkpoint_stop\",\"compact_slow\",\"evict_reposition\"," + "\"checkpoint_evict_page\",\"checkpoint_handle\"," + "\"checkpoint_slow\",\"checkpoint_stop\",\"compact_slow\"," + "\"evict_reposition\"," "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," @@ -1098,8 +1101,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 8}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," - "\"checkpoint_evict_page\",\"checkpoint_slow\"," - "\"checkpoint_stop\",\"compact_slow\",\"evict_reposition\"," + "\"checkpoint_evict_page\",\"checkpoint_handle\"," + "\"checkpoint_slow\",\"checkpoint_stop\",\"compact_slow\"," + "\"evict_reposition\"," "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," @@ -1184,8 +1188,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"tiered_storage", "category", NULL, NULL, confchk_tiered_storage_subconfigs, 8}, {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," - "\"checkpoint_evict_page\",\"checkpoint_slow\"," - "\"checkpoint_stop\",\"compact_slow\",\"evict_reposition\"," + "\"checkpoint_evict_page\",\"checkpoint_handle\"," + "\"checkpoint_slow\",\"checkpoint_stop\",\"compact_slow\"," + "\"evict_reposition\"," "\"failpoint_eviction_fail_after_reconciliation\"," "\"failpoint_history_store_delete_key_from_ts\"," "\"history_store_checkpoint_delay\",\"history_store_search\"," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 90a3841d46d..53646d6d560 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2323,6 +2323,7 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) {"aggressive_sweep", WT_TIMING_STRESS_AGGRESSIVE_SWEEP}, {"backup_rename", WT_TIMING_STRESS_BACKUP_RENAME}, {"checkpoint_evict_page", WT_TIMING_STRESS_CHECKPOINT_EVICT_PAGE}, + {"checkpoint_handle", WT_TIMING_STRESS_CHECKPOINT_HANDLE}, {"checkpoint_slow", WT_TIMING_STRESS_CHECKPOINT_SLOW}, {"checkpoint_stop", WT_TIMING_STRESS_CHECKPOINT_STOP}, {"compact_slow", WT_TIMING_STRESS_COMPACT_SLOW}, diff --git a/src/third_party/wiredtiger/src/include/btree_inline.h b/src/third_party/wiredtiger/src/include/btree_inline.h index 4ca21b0e926..fa43be62765 100644 --- a/src/third_party/wiredtiger/src/include/btree_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_inline.h @@ -1652,7 +1652,8 @@ __wt_page_del_visible(WT_SESSION_IMPL *session, WT_PAGE_DELETED *page_del, bool return (false); } - return (__wt_txn_visible(session, page_del->txnid, page_del->timestamp)); + return ( + __wt_txn_visible(session, page_del->txnid, page_del->timestamp, page_del->durable_timestamp)); } /* @@ -2312,7 +2313,8 @@ __wt_btcur_skip_page( * point added to the page during the last reconciliation. */ if (addr.ta.newest_stop_txn != WT_TXN_MAX && addr.ta.newest_stop_ts != WT_TS_MAX && - __wt_txn_visible(session, addr.ta.newest_stop_txn, addr.ta.newest_stop_ts)) { + __wt_txn_visible(session, addr.ta.newest_stop_txn, addr.ta.newest_stop_ts, + addr.ta.newest_stop_durable_ts)) { *skipp = true; goto unlock; } diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 1523f0cde28..18cb84d1363 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -615,25 +615,26 @@ struct __wt_connection_impl { #define WT_TIMING_STRESS_AGGRESSIVE_SWEEP 0x000001u #define WT_TIMING_STRESS_BACKUP_RENAME 0x000002u #define WT_TIMING_STRESS_CHECKPOINT_EVICT_PAGE 0x000004u -#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x000008u -#define WT_TIMING_STRESS_CHECKPOINT_STOP 0x000010u -#define WT_TIMING_STRESS_COMPACT_SLOW 0x000020u -#define WT_TIMING_STRESS_EVICT_REPOSITION 0x000040u -#define WT_TIMING_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION 0x000080u -#define WT_TIMING_STRESS_FAILPOINT_HISTORY_STORE_DELETE_KEY_FROM_TS 0x000100u -#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x000200u -#define WT_TIMING_STRESS_HS_SEARCH 0x000400u -#define WT_TIMING_STRESS_HS_SWEEP 0x000800u -#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x001000u -#define WT_TIMING_STRESS_SLEEP_BEFORE_READ_OVERFLOW_ONPAGE 0x002000u -#define WT_TIMING_STRESS_SPLIT_1 0x004000u -#define WT_TIMING_STRESS_SPLIT_2 0x008000u -#define WT_TIMING_STRESS_SPLIT_3 0x010000u -#define WT_TIMING_STRESS_SPLIT_4 0x020000u -#define WT_TIMING_STRESS_SPLIT_5 0x040000u -#define WT_TIMING_STRESS_SPLIT_6 0x080000u -#define WT_TIMING_STRESS_SPLIT_7 0x100000u -#define WT_TIMING_STRESS_TIERED_FLUSH_FINISH 0x200000u +#define WT_TIMING_STRESS_CHECKPOINT_HANDLE 0x000008u +#define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x000010u +#define WT_TIMING_STRESS_CHECKPOINT_STOP 0x000020u +#define WT_TIMING_STRESS_COMPACT_SLOW 0x000040u +#define WT_TIMING_STRESS_EVICT_REPOSITION 0x000080u +#define WT_TIMING_STRESS_FAILPOINT_EVICTION_FAIL_AFTER_RECONCILIATION 0x000100u +#define WT_TIMING_STRESS_FAILPOINT_HISTORY_STORE_DELETE_KEY_FROM_TS 0x000200u +#define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x000400u +#define WT_TIMING_STRESS_HS_SEARCH 0x000800u +#define WT_TIMING_STRESS_HS_SWEEP 0x001000u +#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x002000u +#define WT_TIMING_STRESS_SLEEP_BEFORE_READ_OVERFLOW_ONPAGE 0x004000u +#define WT_TIMING_STRESS_SPLIT_1 0x008000u +#define WT_TIMING_STRESS_SPLIT_2 0x010000u +#define WT_TIMING_STRESS_SPLIT_3 0x020000u +#define WT_TIMING_STRESS_SPLIT_4 0x040000u +#define WT_TIMING_STRESS_SPLIT_5 0x080000u +#define WT_TIMING_STRESS_SPLIT_6 0x100000u +#define WT_TIMING_STRESS_SPLIT_7 0x200000u +#define WT_TIMING_STRESS_TIERED_FLUSH_FINISH 0x400000u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 1c56ccc7538..3f4ad155e94 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -2084,8 +2084,8 @@ static inline bool __wt_txn_upd_visible(WT_SESSION_IMPL *session, WT_UPDATE *upd WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_txn_upd_visible_all(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline bool __wt_txn_visible(WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline bool __wt_txn_visible(WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp, + wt_timestamp_t durable_timestamp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_txn_visible_all(WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_txn_visible_id_snapshot(uint64_t id, uint64_t snap_min, uint64_t snap_max, diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index d1ede81e211..6140535c7cc 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -295,6 +295,7 @@ struct __wt_txn { * the current oldest/pinned timestamp, respectively. */ wt_timestamp_t checkpoint_read_timestamp; + wt_timestamp_t checkpoint_stable_timestamp; wt_timestamp_t checkpoint_oldest_timestamp; /* Array of modifications by this transaction. */ diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index f6a75c9c785..51b7147f7b4 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -677,7 +677,7 @@ static inline bool __wt_txn_tw_stop_visible(WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw) { return (WT_TIME_WINDOW_HAS_STOP(tw) && !tw->prepare && - __wt_txn_visible(session, tw->stop_txn, tw->stop_ts)); + __wt_txn_visible(session, tw->stop_txn, tw->stop_ts, tw->durable_stop_ts)); } /* @@ -695,7 +695,7 @@ __wt_txn_tw_start_visible(WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw) (tw->start_txn != tw->stop_txn || tw->start_ts != tw->stop_ts || tw->durable_start_ts != tw->durable_stop_ts)) || !tw->prepare) && - __wt_txn_visible(session, tw->start_txn, tw->start_ts)); + __wt_txn_visible(session, tw->start_txn, tw->start_ts, tw->durable_start_ts)); } /* @@ -797,7 +797,8 @@ __txn_visible_id(WT_SESSION_IMPL *session, uint64_t id) * Can the current transaction see the given ID / timestamp? */ static inline bool -__wt_txn_visible(WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp) +__wt_txn_visible( + WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp, wt_timestamp_t durable_timestamp) { WT_TXN *txn; WT_TXN_SHARED *txn_shared; @@ -816,8 +817,17 @@ __wt_txn_visible(WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp if (!F_ISSET(txn, WT_TXN_SHARED_TS_READ) || timestamp == WT_TS_NONE) return (true); + /* + * For checkpoint cursors, just using the commit timestamp visibility check can go wrong when a + * prepared transaction gets committed in parallel to a running checkpoint. + * + * To avoid this problem, along with the visibility check of a commit timestamp, comparing the + * durable timestamp against the stable timestamp of a checkpoint can avoid the problems of + * returning inconsistent data. + */ if (WT_READING_CHECKPOINT(session)) - return (timestamp <= txn->checkpoint_read_timestamp); + return ((timestamp <= txn->checkpoint_read_timestamp) && + (durable_timestamp <= txn->checkpoint_stable_timestamp)); return (timestamp <= txn_shared->read_timestamp); } @@ -843,7 +853,7 @@ __wt_txn_upd_visible_type(WT_SESSION_IMPL *session, WT_UPDATE *upd) upd->type == WT_UPDATE_STANDARD)) return (WT_VISIBLE_TRUE); - upd_visible = __wt_txn_visible(session, upd->txnid, upd->start_ts); + upd_visible = __wt_txn_visible(session, upd->txnid, upd->start_ts, upd->durable_ts); /* * The visibility check is only valid if the update does not change state. If the state does diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index c64733669ce..e54467c0f32 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1409,7 +1409,8 @@ __clsm_put(WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm, const WT_ITEM *key, co for (i = 0, slot = clsm->nchunks - 1; i < clsm->nupdates; i++, slot--) { /* Check if we need to keep updating old chunks. */ - if (i > 0 && __wt_txn_visible(session, clsm->chunks[slot]->switch_txn, WT_TS_NONE)) { + if (i > 0 && + __wt_txn_visible(session, clsm->chunks[slot]->switch_txn, WT_TS_NONE, WT_TS_NONE)) { clsm->nupdates = i; break; } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index a51d05058f6..db48ca8e045 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -2167,8 +2167,9 @@ __wt_txn_init_checkpoint_cursor( */ snapinfo->snapshot_txns = NULL; - /* Set the read and oldest timestamps. */ + /* Set the read, stable and oldest timestamps. */ txn->checkpoint_read_timestamp = snapinfo->stable_ts; + txn->checkpoint_stable_timestamp = snapinfo->stable_ts; txn->checkpoint_oldest_timestamp = snapinfo->oldest_ts; /* Set the flag that indicates if we have a timestamp. */ diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 3e668feb5d9..03a4ebb58fc 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -2318,6 +2318,7 @@ __checkpoint_presync(WT_SESSION_IMPL *session, const char *cfg[]) static int __checkpoint_tree_helper(WT_SESSION_IMPL *session, const char *cfg[]) { + struct timespec tsp; WT_BTREE *btree; WT_DECL_RET; WT_TXN *txn; @@ -2326,6 +2327,11 @@ __checkpoint_tree_helper(WT_SESSION_IMPL *session, const char *cfg[]) btree = S2BT(session); txn = session->txn; + /* Add a two seconds wait to simulate checkpoint slowness for every handle. */ + tsp.tv_sec = 2; + tsp.tv_nsec = 0; + __checkpoint_timing_stress(session, WT_TIMING_STRESS_CHECKPOINT_HANDLE, &tsp); + /* Are we using a read timestamp for this checkpoint transaction? */ with_timestamp = F_ISSET(txn, WT_TXN_SHARED_TS_READ); diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint21.py b/src/third_party/wiredtiger/test/suite/test_checkpoint21.py index b43060da2b3..aa5822b2fd0 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint21.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint21.py @@ -177,8 +177,8 @@ class test_checkpoint(wttest.WiredTigerTestCase): # Read the checkpoint. self.check(ds, self.first_checkpoint, nrows, value_a, 15) - self.check(ds, self.first_checkpoint, nrows, value_b, 25) - self.check(ds, self.first_checkpoint, nrows, value_b, None) # default read ts + self.check(ds, self.first_checkpoint, nrows, value_a, 25) + self.check(ds, self.first_checkpoint, nrows, value_a, None) # default read ts self.check(ds, self.first_checkpoint, nrows, value_b, 0) # no read ts if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint28.py b/src/third_party/wiredtiger/test/suite/test_checkpoint28.py new file mode 100644 index 00000000000..e502b6fe719 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint28.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import threading, time +import wttest +import wiredtiger +from wiredtiger import stat +from wtthread import checkpoint_thread +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_checkpoint28.py +# +# Make sure that when we open a cursor we secure the proper matching +# history store checkpoint, and don't bobble or lose it if the database +# moves on. Non-timestamp version. +# +# It doesn't make sense to run this test for named checkpoints, because +# regenerating a named checkpoint with the cursor open isn't allowed and +# generating two different checkpoints with different names doesn't make +# an interesting scenario. The concern is getting the matching version +# of WiredTigerCheckpoint and hanging onto it. + +class test_checkpoint(wttest.WiredTigerTestCase): + conn_config = 'statistics=(all),timing_stress_for_test=[checkpoint_handle]' + session_config = 'isolation=snapshot' + + format_values = [ + ('column-fix', dict(key_format='r', value_format='8t', + extraconfig=',allocation_size=512,leaf_page_max=512')), + ('column', dict(key_format='r', value_format='S', extraconfig='')), + ('string_row', dict(key_format='S', value_format='S', extraconfig='')), + ] + scenarios = make_scenarios(format_values) + + def large_updates(self, ds, nrows, value): + cursor = self.session.open_cursor(ds.uri) + self.session.begin_transaction() + for i in range(1, nrows + 1): + cursor[ds.key(i)] = value + if i % 101 == 0: + self.session.commit_transaction() + self.session.begin_transaction() + self.session.commit_transaction() + cursor.close() + + def check(self, cursor, value): + count = 0 + zero_count = 0 + for k, v in cursor: + if self.value_format == '8t' and v == 0: + zero_count += 1 + else: + self.assertEqual(v, value) + count += 1 + return count + + def test_checkpoint(self): + uri_1 = 'table:checkpoint28_1' + uri_2 = 'table:checkpoint28_2' + nrows = 1000 + + # Create two tables. + ds_1 = SimpleDataSet( + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format, + config=self.extraconfig) + ds_1.populate() + + ds_2 = SimpleDataSet( + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format, + config=self.extraconfig) + ds_2.populate() + + if self.value_format == '8t': + nrows *= 5 + value_a = 97 + else: + value_a = "aaaaa" * 100 + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + # Write more data. + session2 = self.conn.open_session() + cursor1 = session2.open_cursor(uri_1) + cursor2 = session2.open_cursor(uri_2) + session2.begin_transaction() + for i in range(1, nrows+1): + cursor1[ds_1.key(i)] = value_a + cursor2[ds_2.key(i)] = value_a + session2.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) + + # Move stable up to 30. + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30)) + + # Commit the transaction with a background checkpoint so we get part of it + # in the checkpoint. + done = threading.Event() + ckpt = checkpoint_thread(self.conn, done) + try: + ckpt.start() + + # Wait for checkpoint to start before committing. + ckpt_started = 0 + while not ckpt_started: + stat_cursor = self.session.open_cursor('statistics:', None, None) + ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] + stat_cursor.close() + time.sleep(1) + + # Commit the transaction at 25 but make it durable at 35. + session2.timestamp_transaction('commit_timestamp=' + self.timestamp_str(25)) + session2.timestamp_transaction('durable_timestamp=' + self.timestamp_str(35)) + session2.commit_transaction() + finally: + done.set() + ckpt.join() + + # Open the checkpoint now. + ckpt_1 = self.session.open_cursor(uri_1, None, 'checkpoint=WiredTigerCheckpoint') + ckpt_2 = self.session.open_cursor(uri_2, None, 'checkpoint=WiredTigerCheckpoint') + + # Make sure we can't read any of the rows from two tables. + uri_1_count = self.check(ckpt_1, value_a) + uri_2_count = self.check(ckpt_2, value_a) + + self.assertEqual(uri_1_count, uri_2_count) + self.assertEqual(uri_1_count, 0) + + ckpt_1.close() + ckpt_2.close() + +if __name__ == '__main__': + wttest.run() |