From 1abfd89c2c2c7fefb1438f54f54a9daa1b84d706 Mon Sep 17 00:00:00 2001 From: Benety Goh Date: Fri, 20 Aug 2021 15:18:03 -0400 Subject: Revert "Import wiredtiger: b77c2015c6c7a98cf06f7cef7e11bd1bbe686142 from branch mongodb-master" This reverts commit 75d04d3757a9cea8e0d712e05afa6174b5df4892. Revert "Import wiredtiger: 73489047a5b56814b93c790ee04e24d84d49510e from branch mongodb-master" This reverts commit b6ead6db6ea45e5b9b6a549edc9dae1ae20011bc. Revert "Import wiredtiger: 9149c005b6c6789dae09de1911c3669f63527b00 from branch mongodb-master" This reverts commit 1c41fd963dd4f15c5f108932ad36ae4056f5ca33. Revert "Import wiredtiger: 79d24845314f944c9889e1ddc199bd6857977b3a from branch mongodb-master" This reverts commit ade843756b2e8638a01f6bc6ace63c8195af46e3. Revert "Import wiredtiger: 797f085d4af8abea62dd2dd4da5f551c48a98067 from branch mongodb-master" This reverts commit 7dd3891b50e0ade5281656960d240c1db3f7934f. --- src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/src/include/cell_inline.h | 18 ++ src/third_party/wiredtiger/src/include/extern.h | 4 - src/third_party/wiredtiger/src/meta/meta_ckpt.c | 34 --- src/third_party/wiredtiger/src/reconcile/rec_col.c | 1 + src/third_party/wiredtiger/src/reconcile/rec_row.c | 5 +- .../wiredtiger/src/reconcile/rec_write.c | 6 +- src/third_party/wiredtiger/src/txn/txn.c | 40 --- src/third_party/wiredtiger/src/txn/txn_ckpt.c | 8 - src/third_party/wiredtiger/src/txn/txn_recover.c | 44 +++- .../wiredtiger/src/txn/txn_rollback_to_stable.c | 132 +++++++--- .../wiredtiger/test/checkpoint/checkpointer.c | 18 +- .../wiredtiger/test/checkpoint/test_checkpoint.c | 3 +- src/third_party/wiredtiger/test/csuite/Makefile.am | 9 +- .../wiredtiger/test/csuite/schema_abort/main.c | 54 ++-- .../wiredtiger/test/csuite/schema_abort/smoke.sh | 8 - .../wiredtiger/test/csuite/timestamp_abort/main.c | 77 ++---- .../test/csuite/timestamp_abort/smoke.sh | 4 - .../wiredtiger/test/csuite/truncated_log/main.c | 25 +- .../wiredtiger/test/csuite/truncated_log/smoke.sh | 20 -- .../wiredtiger/test/csuite/wt6185_modify_ts/main.c | 60 +---- .../test/csuite/wt6185_modify_ts/smoke.sh | 21 -- .../test/csuite/wt6616_checkpoint_oldest_ts/main.c | 30 +-- .../csuite/wt6616_checkpoint_oldest_ts/smoke.sh | 21 -- src/third_party/wiredtiger/test/evergreen.yml | 12 - src/third_party/wiredtiger/test/format/config.c | 2 - src/third_party/wiredtiger/test/suite/test_hs18.py | 28 +- .../test/suite/test_rollback_to_stable22.py | 2 + .../test/suite/test_rollback_to_stable24.py | 135 ---------- .../test/suite/test_rollback_to_stable25.py | 293 --------------------- 30 files changed, 254 insertions(+), 862 deletions(-) delete mode 100755 src/third_party/wiredtiger/test/csuite/truncated_log/smoke.sh delete mode 100755 src/third_party/wiredtiger/test/csuite/wt6185_modify_ts/smoke.sh delete mode 100755 src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/smoke.sh delete mode 100644 src/third_party/wiredtiger/test/suite/test_rollback_to_stable24.py delete mode 100644 src/third_party/wiredtiger/test/suite/test_rollback_to_stable25.py diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 03ee1234a25..d3a515b6cc0 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "b77c2015c6c7a98cf06f7cef7e11bd1bbe686142" + "commit": "d384ce4385d1827e36fd4eeed893cc9b45b42111" } diff --git a/src/third_party/wiredtiger/src/include/cell_inline.h b/src/third_party/wiredtiger/src/include/cell_inline.h index 97b856e8a62..2d91b9a8ee6 100644 --- a/src/third_party/wiredtiger/src/include/cell_inline.h +++ b/src/third_party/wiredtiger/src/include/cell_inline.h @@ -1044,6 +1044,24 @@ __cell_unpack_window_cleanup(WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk __cell_kv_window_cleanup(session, unpack_kv); } +/* + * __cell_pack_kv_window_cleanup -- + * Clean up cells loaded from a previous run while writing to disk. + */ +static inline void +__cell_pack_kv_window_cleanup( + WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, WT_CELL_UNPACK_KV *unpack_kv) +{ + /* + * If the page came from a previous run, reset the transaction ids to "none" and timestamps to 0 + * as appropriate when the cell information is used for packing the new cell. + */ + if (F_ISSET(S2C(session), WT_CONN_RECOVERING) && + dsk->write_gen > S2BT(session)->base_write_gen && + dsk->write_gen < S2BT(session)->run_write_gen) + __cell_kv_window_cleanup(session, unpack_kv); +} + /* * __wt_cell_unpack_addr -- * Unpack an address WT_CELL into a structure. diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 8061ce88008..5d982b43d09 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -39,8 +39,6 @@ extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_txn_active(WT_SESSION_IMPL *session, uint64_t txnid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern bool __wt_txn_user_active(WT_SESSION_IMPL *session) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern char *__wt_time_aggregate_to_string(WT_TIME_AGGREGATE *ta, char *ta_string) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern char *__wt_time_point_to_string(wt_timestamp_t ts, wt_timestamp_t durable_ts, @@ -1090,8 +1088,6 @@ extern int __wt_meta_track_update(WT_SESSION_IMPL *session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_btree_id_to_uri(WT_SESSION_IMPL *session, uint32_t btree_id, char **uri) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_metadata_correct_base_write_gen(WT_SESSION_IMPL *session) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor_close(WT_SESSION_IMPL *session) diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index d58d0149658..b4c7c933c62 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -973,40 +973,6 @@ err: return (ret); } -/* - * __wt_metadata_correct_base_write_gen -- - * Update the connection's base write generation from all files in metadata at then end of the - * recovery checkpoint. - */ -int -__wt_metadata_correct_base_write_gen(WT_SESSION_IMPL *session) -{ - WT_CURSOR *cursor; - WT_DECL_RET; - char *config, *uri; - - uri = NULL; - WT_RET(__wt_metadata_cursor(session, &cursor)); - while ((ret = cursor->next(cursor)) == 0) { - WT_ERR(cursor->get_key(cursor, &uri)); - - if (!WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "tiered:")) - continue; - - WT_ERR(cursor->get_value(cursor, &config)); - - /* Update base write gen to the write gen. */ - WT_ERR(__wt_metadata_update_base_write_gen(session, config)); - } - WT_ERR_NOTFOUND_OK(ret, false); - -err: - if (ret != 0 && uri != NULL) - __wt_err(session, ret, "unable to correct write gen for %s", uri); - WT_TRET(__wt_metadata_cursor_release(session, &cursor)); - return (ret); -} - /* * __wt_meta_ckptlist_to_meta -- * Convert a checkpoint list into its metadata representation. diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 8000026c58b..4ea57b8acc7 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -744,6 +744,7 @@ record_loop: twp = &clear_tw; goto compare; } + __cell_pack_kv_window_cleanup(session, page->dsk, vpack); twp = &vpack->tw; /* diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 99d887da573..a72bc170245 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -808,9 +808,10 @@ __wt_rec_row_leaf( upd = upd_select.upd; /* Take the timestamp from the update or the cell. */ - if (upd == NULL) + if (upd == NULL) { + __cell_pack_kv_window_cleanup(session, page->dsk, vpack); twp = &vpack->tw; - else + } else twp = &upd_select.tw; /* diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 3971d287550..9ad305792b9 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -1516,7 +1516,7 @@ err: * Initialize the page write generation number. */ static void -__rec_set_page_write_gen(WT_BTREE *btree, WT_PAGE_HEADER *dsk) +__rec_set_page_write_gen(WT_PAGE_HEADER *dsk, WT_BTREE *btree) { /* * We increment the block's write generation so it's easy to identify newer versions of blocks @@ -1553,7 +1553,7 @@ __rec_split_write_header(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_CHUNK dsk->recno = btree->type == BTREE_ROW ? WT_RECNO_OOB : multi->key.recno; - __rec_set_page_write_gen(btree, dsk); + __rec_set_page_write_gen(dsk, btree); dsk->mem_size = multi->size; dsk->u.entries = chunk->entries; dsk->type = page->type; @@ -2367,7 +2367,7 @@ __wt_rec_cell_build_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_KV *k dsk = tmp->mem; memset(dsk, 0, WT_PAGE_HEADER_SIZE); dsk->type = WT_PAGE_OVFL; - __rec_set_page_write_gen(btree, dsk); + __rec_set_page_write_gen(dsk, btree); dsk->u.datalen = (uint32_t)kv->buf.size; memcpy(WT_PAGE_HEADER_BYTE(btree, dsk), kv->buf.data, kv->buf.size); dsk->mem_size = WT_PAGE_HEADER_BYTE_SIZE(btree) + (uint32_t)kv->buf.size; diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 7f05aab6cef..35eb2e08d6f 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -138,46 +138,6 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) } } -/* - * __wt_txn_user_active -- - * Check whether there are any running user transactions. Note that a new transactions may start - * on a session we have already examined and the caller needs to be aware of this limitation. - * Exclude prepared user transactions from this check. - */ -bool -__wt_txn_user_active(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - WT_SESSION_IMPL *session_in_list; - uint32_t i, session_cnt; - bool txn_active; - - conn = S2C(session); - txn_active = false; - - /* - * No lock is required because the session array is fixed size, but it may contain inactive - * entries. We must review any active session, so insert a read barrier after reading the active - * session count. That way, no matter what sessions come or go, we'll check the slots for all of - * the user sessions for active transactions when we started our check. - */ - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (i = 0; i < session_cnt; i++) { - WT_STAT_CONN_INCR(session, txn_sessions_walked); - session_in_list = &conn->sessions[i]; - /* Check if a user session has a running transaction. Ignore prepared transactions. */ - if (F_ISSET(session_in_list->txn, WT_TXN_RUNNING) && - !F_ISSET(session_in_list, WT_SESSION_INTERNAL) && - !F_ISSET(session_in_list->txn, WT_TXN_PREPARE)) { - - txn_active = true; - break; - } - } - - return (txn_active); -} - /* * __wt_txn_active -- * Check if a transaction is still active. If not, it is either committed, prepared, or rolled diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index c32cd43b0bc..1a7090cd2c6 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -921,14 +921,6 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_STAT_CONN_SET(session, txn_hs_ckpt_duration, hs_ckpt_duration_usecs); } - /* - * As part of recovery, rollback to stable may have left out clearing stale transaction ids. - * Update the connection base write generation based on the latest checkpoint write generations - * to reset these transaction ids present on the pages when reading them. - */ - if (F_ISSET(conn, WT_CONN_RECOVERING)) - WT_ERR(__wt_metadata_correct_base_write_gen(session)); - /* * Clear the dhandle so the visibility check doesn't get confused about the snap min. Don't * bother restoring the handle since it doesn't make sense to carry a handle across a diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index c76e2af3597..9f32e9346f0 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -547,6 +547,39 @@ err: return (ret); } +/* + * __recovery_correct_write_gen -- + * Update the connection's base write generation from all files in metadata. + */ +static int +__recovery_correct_write_gen(WT_SESSION_IMPL *session) +{ + WT_CURSOR *cursor; + WT_DECL_RET; + char *config, *uri; + + uri = NULL; + WT_RET(__wt_metadata_cursor(session, &cursor)); + while ((ret = cursor->next(cursor)) == 0) { + WT_ERR(cursor->get_key(cursor, &uri)); + + if (!WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "tiered:")) + continue; + + WT_ERR(cursor->get_value(cursor, &config)); + + /* Update base write gen to the write gen. */ + WT_ERR(__wt_metadata_update_base_write_gen(session, config)); + } + WT_ERR_NOTFOUND_OK(ret, false); + +err: + if (ret != 0 && uri != NULL) + __wt_err(session, ret, "unable to correct write gen for %s", uri); + WT_TRET(__wt_metadata_cursor_release(session, &cursor)); + return (ret); +} + /* * __recovery_setup_file -- * Set up the recovery slot for a file, track the largest file ID, and update the base write gen @@ -1021,12 +1054,17 @@ done: */ WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); + /* + * Rollback to stable may have left out clearing stale transaction ids. Update the connection + * base write generation based on the latest checkpoint write generations to reset them. + */ + if (rts_executed) + WT_ERR(__recovery_correct_write_gen(session)); + /* * Update the open dhandles write generations and base write generation with the connection's * base write generation because the recovery checkpoint writes the pages to disk with new write - * generation number which contains transaction ids that are needed to reset later. The - * connection level base write generation number is updated at the end of the recovery - * checkpoint. + * generation number which contains transaction ids that are needed to reset later. */ __wt_dhandle_update_write_gens(session); diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 6004ddd3db2..5acd3ca398a 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -594,9 +594,6 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page /* Finally remove that update from history store. */ if (valid_update_found) { - /* Avoid freeing the updates while still in use if hs_cursor->remove fails. */ - upd = tombstone = NULL; - WT_ERR(hs_cursor->remove(hs_cursor)); WT_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); WT_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts); @@ -788,7 +785,7 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r WT_PAGE *page; uint64_t recno, rle; uint32_t i, j; - bool is_ondisk_stable, stable_update_found; + bool stable_update_found; page = ref->page; /* @@ -807,46 +804,26 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r WT_RET(__rollback_abort_insert_list( session, page, ins, rollback_timestamp, &stable_update_found)); - if (page->dsk != NULL) { - /* Unpack the cell. We need its RLE count whether or not we're going to iterate it. */ + if (!stable_update_found && page->dsk != NULL) { kcell = WT_COL_PTR(page, cip); __wt_cell_unpack_kv(session, page->dsk, kcell, &unpack); rle = __wt_cell_rle(&unpack); - - /* - * If we found a stable update on the insert list, this key needs no further attention. - * Any other keys in this cell with stable updates also do not require attention. But - * beyond that, the on-disk value must be older than - * the update we found. That means it too is stable(*), so any keys in the cell that - * _don't_ have stable updates on the update list don't need further attention either. - * (And any unstable updates were just handled above.) Thus we can skip iterating over - * the cell. - * - * Furthermore, if the cell is deleted it must be - * itself stable, because cells only appear as deleted if there is no older value that - * might need to be restored. We can skip iterating over the cell. - * - * (*) Either that, or the update is not timestamped, in which case the on-disk value - * might not be stable but the non-timestamp update will hide it until the next - * reconciliation and then overwrite it. - */ - if (stable_update_found) - WT_STAT_CONN_DATA_INCR(session, txn_rts_stable_rle_skipped); - else if (unpack.type == WT_CELL_DEL) - WT_STAT_CONN_DATA_INCR(session, txn_rts_delete_rle_skipped); - else { + if (unpack.type != WT_CELL_DEL) { for (j = 0; j < rle; j++) { - WT_RET(__rollback_abort_ondisk_kv( - session, ref, cip, NULL, rollback_timestamp, recno + j, &is_ondisk_stable)); - /* We can stop right away if the on-disk version is stable. */ - if (is_ondisk_stable) { + WT_RET(__rollback_abort_ondisk_kv(session, ref, cip, NULL, rollback_timestamp, + recno + j, &stable_update_found)); + /* Skip processing all RLE if the on-disk version is stable. */ + if (stable_update_found) { if (rle > 1) WT_STAT_CONN_DATA_INCR(session, txn_rts_stable_rle_skipped); break; } } - } + } else + WT_STAT_CONN_DATA_INCR(session, txn_rts_delete_rle_skipped); recno += rle; + } else { + recno++; } } @@ -1217,16 +1194,17 @@ __rollback_to_stable_check(WT_SESSION_IMPL *session) bool txn_active; /* - * Help the user comply with the requirement that there are no concurrent user operations. It is - * okay to have a transaction in prepared state. + * Help the user comply with the requirement that there are no concurrent operations. Protect + * against spurious conflicts with the sweep server: we exclude it from running concurrent with + * rolling back the history store contents. */ - txn_active = __wt_txn_user_active(session); + ret = __wt_txn_activity_check(session, &txn_active); #ifdef HAVE_DIAGNOSTIC if (txn_active) WT_TRET(__wt_verbose_dump_txn(session)); #endif - if (txn_active) + if (ret == 0 && txn_active) WT_RET_MSG(session, EINVAL, "rollback_to_stable illegal with active transactions"); return (ret); @@ -1624,15 +1602,84 @@ err: static int __rollback_to_stable(WT_SESSION_IMPL *session, bool no_ckpt) { + WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_TXN_GLOBAL *txn_global; wt_timestamp_t rollback_timestamp; + size_t retries; + uint32_t cache_flags; char ts_string[2][WT_TS_INT_STRING_SIZE]; conn = S2C(session); + cache = conn->cache; txn_global = &conn->txn_global; + /* + * We're about to run a check for active transactions in the system to stop users from shooting + * themselves in the foot. Eviction threads may interfere with this check if they involve writes + * to the history store so we need to wait until the system is no longer evicting content. + * + * If we detect active evictions, we should wait a millisecond and check again. If we're waiting + * for evictions to quiesce for more than 2 minutes, we should give up on waiting and proceed + * with the transaction check anyway. + */ +#define WT_RTS_EVICT_MAX_RETRIES (2 * WT_MINUTE * WT_THOUSAND) + /* + * These are the types of evictions that can result in a history store operation. Since we want + * to avoid these happening concurrently with our check, we need to look for these flags. + */ +#define WT_CACHE_EVICT_HS_FLAGS \ + (WT_CACHE_EVICT_DIRTY | WT_CACHE_EVICT_UPDATES | WT_CACHE_EVICT_URGENT) + for (retries = 0; retries < WT_RTS_EVICT_MAX_RETRIES; ++retries) { + /* + * If we're shutting down or running with an in-memory configuration, we aren't at risk of + * racing with history store transactions. + */ + if (F_ISSET(conn, WT_CONN_CLOSING_TIMESTAMP | WT_CONN_IN_MEMORY)) + break; + + /* Check whether eviction has quiesced. */ + WT_ORDERED_READ(cache_flags, cache->flags); + if (!FLD_ISSET(cache_flags, WT_CACHE_EVICT_HS_FLAGS)) { + /* + * If we we find that the eviction flags are unset, interrupt the eviction server and + * acquire the pass lock to stop the server from setting the eviction flags AFTER this + * point and racing with our check. + */ + (void)__wt_atomic_addv32(&cache->pass_intr, 1); + __wt_spin_lock(session, &cache->evict_pass_lock); + (void)__wt_atomic_subv32(&cache->pass_intr, 1); + FLD_SET(session->lock_flags, WT_SESSION_LOCKED_PASS); + + /* + * Check that the flags didn't get set in between when we checked and when we acquired + * the server lock. If it did get set, release the locks and keep trying. If they're + * still unset, break out of this loop and commence our check. + */ + WT_ORDERED_READ(cache_flags, cache->flags); + if (!FLD_ISSET(cache_flags, WT_CACHE_EVICT_HS_FLAGS)) + break; + else { + __wt_spin_unlock(session, &cache->evict_pass_lock); + FLD_CLR(session->lock_flags, WT_SESSION_LOCKED_PASS); + } + } + /* If we're retrying, pause for a millisecond and let eviction make some progress. */ + __wt_sleep(0, WT_THOUSAND); + } + if (retries == WT_RTS_EVICT_MAX_RETRIES) { + WT_ERR(__wt_msg( + session, "timed out waiting for eviction to quiesce, running rollback to stable")); + /* + * FIXME: WT-7877 RTS fails when there are active transactions running in parallel to it. + * Waiting in a loop for eviction to quiesce is not efficient in some scenarios where the + * cache is not cleared in 2 minutes. Enable the following assert and + * test_rollback_to_stable22.py when the cache issue is addressed. + */ + /* WT_ASSERT(session, false && "Timed out waiting for eviction to quiesce prior to rts"); */ + } + /* * Rollback to stable should ignore tombstones in the history store since it needs to scan the * entire table sequentially. @@ -1641,6 +1688,11 @@ __rollback_to_stable(WT_SESSION_IMPL *session, bool no_ckpt) WT_ERR(__rollback_to_stable_check(session)); + if (FLD_ISSET(session->lock_flags, WT_SESSION_LOCKED_PASS)) { + __wt_spin_unlock(session, &cache->evict_pass_lock); + FLD_CLR(session->lock_flags, WT_SESSION_LOCKED_PASS); + } + /* * Copy the stable timestamp, otherwise we'd need to lock it each time it's accessed. Even * though the stable timestamp isn't supposed to be updated while rolling back, accessing it @@ -1674,6 +1726,10 @@ __rollback_to_stable(WT_SESSION_IMPL *session, bool no_ckpt) WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); err: + if (FLD_ISSET(session->lock_flags, WT_SESSION_LOCKED_PASS)) { + __wt_spin_unlock(session, &cache->evict_pass_lock); + FLD_CLR(session->lock_flags, WT_SESSION_LOCKED_PASS); + } F_CLR(session, WT_SESSION_ROLLBACK_TO_STABLE); return (ret); } diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index a2445225e2e..b7e7a9a952b 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -35,19 +35,6 @@ static int diagnose_key_error(WT_CURSOR *, int, WT_CURSOR *, int); static int real_checkpointer(void); static int verify_consistency(WT_SESSION *, char *); -/* - * set_stable -- - * Set the stable timestamp from g.ts_stable. - */ -static void -set_stable(void) -{ - char buf[128]; - - testutil_check(__wt_snprintf(buf, sizeof(buf), "stable_timestamp=%x", g.ts_stable)); - testutil_check(g.conn->set_timestamp(g.conn, buf)); -} - /* * start_checkpoints -- * Responsible for creating the checkpoint thread. @@ -55,7 +42,6 @@ set_stable(void) void start_checkpoints(void) { - set_stable(); testutil_check(__wt_thread_create(NULL, &g.checkpoint_thread, checkpointer, NULL)); if (g.use_timestamps) { testutil_check(__wt_rwlock_init(NULL, &g.clock_lock)); @@ -88,6 +74,7 @@ clock_thread(void *arg) WT_SESSION *wt_session; WT_SESSION_IMPL *session; uint64_t delay; + char buf[128]; WT_UNUSED(arg); @@ -98,7 +85,8 @@ clock_thread(void *arg) while (g.running) { __wt_writelock(session, &g.clock_lock); ++g.ts_stable; - set_stable(); + testutil_check(__wt_snprintf(buf, sizeof(buf), "stable_timestamp=%x", g.ts_stable)); + testutil_check(g.conn->set_timestamp(g.conn, buf)); if (g.ts_stable % 997 == 0) { /* * Random value between 6 and 10 seconds. diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 1d4c99a2b03..90378cc0de9 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -140,8 +140,7 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(g.home, 512, working_dir); - /* Start time at 1 since 0 is not a valid timestamp. */ - g.ts_stable = 1; + g.ts_stable = 0; printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index f5da81e75ed..f5c3eaed361 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -37,7 +37,7 @@ all_TESTS += timestamp_abort/smoke.sh test_truncated_log_SOURCES = truncated_log/main.c noinst_PROGRAMS += test_truncated_log -all_TESTS += truncated_log/smoke.sh +all_TESTS += test_truncated_log test_wt1965_col_efficiency_SOURCES = wt1965_col_efficiency/main.c noinst_PROGRAMS += test_wt1965_col_efficiency @@ -49,7 +49,8 @@ all_TESTS += test_wt2403_lsm_workload test_wt2246_col_append_SOURCES = wt2246_col_append/main.c noinst_PROGRAMS += test_wt2246_col_append -all_TESTS += test_wt2246_col_append +# Temporarily disabled (WT-5790) +# all_TESTS += test_wt2246_col_append test_wt2323_join_visibility_SOURCES = wt2323_join_visibility/main.c noinst_PROGRAMS += test_wt2323_join_visibility @@ -145,11 +146,11 @@ all_TESTS += test_wt4891_meta_ckptlist_get_alloc test_wt6185_modify_ts_SOURCES = wt6185_modify_ts/main.c noinst_PROGRAMS += test_wt6185_modify_ts -all_TESTS += wt6185_modify_ts/smoke.sh +all_TESTS += test_wt6185_modify_ts test_wt6616_checkpoint_oldest_ts_SOURCES = wt6616_checkpoint_oldest_ts/main.c noinst_PROGRAMS += test_wt6616_checkpoint_oldest_ts -all_TESTS += wt6616_checkpoint_oldest_ts/smoke.sh +all_TESTS += test_wt6616_checkpoint_oldest_ts # Run this during a "make check" smoke test. TESTS = $(all_TESTS) diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index acf14348c0a..e16c7d7e79e 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -75,7 +75,7 @@ static const char *const uri_collection = "table:collection"; static const char *const ckpt_file = "checkpoint_done"; -static bool compat, inmem, stable_set, use_columns, use_ts, use_txn; +static bool compat, inmem, stable_set, use_ts, use_txn; static volatile uint64_t global_ts = 1; static volatile uint64_t uid = 1; typedef struct { @@ -96,10 +96,9 @@ static volatile THREAD_TS th_ts[MAX_TH]; /* * A minimum width of 10, along with zero filling, means that all the keys sort according to their - * integer value, making each thread's key space distinct. For column-store we just use the integer - * values and that has the same effect. + * integer value, making each thread's key space distinct. */ -#define ROW_KEY_FORMAT ("%010" PRIu64) +#define KEY_FORMAT ("%010" PRIu64) typedef struct { uint64_t absent_key; /* Last absent key */ @@ -671,20 +670,14 @@ thread_run(void *arg) } if (use_ts) stable_ts = __wt_atomic_addv64(&global_ts, 1); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, i)); testutil_check(session->begin_transaction(session, NULL)); if (use_prep) testutil_check(oplog_session->begin_transaction(oplog_session, NULL)); - if (use_columns) { - cur_coll->set_key(cur_coll, i + 1); - cur_local->set_key(cur_local, i + 1); - cur_oplog->set_key(cur_oplog, i + 1); - } else { - testutil_check(__wt_snprintf(kname, sizeof(kname), ROW_KEY_FORMAT, i)); - cur_coll->set_key(cur_coll, kname); - cur_local->set_key(cur_local, kname); - cur_oplog->set_key(cur_oplog, kname); - } + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); /* * Put an informative string into the value so that it can be viewed well in a binary dump. */ @@ -771,7 +764,7 @@ run_workload(uint32_t nth) THREAD_DATA *td; wt_thread_t *thr; uint32_t ckpt_id, i, ts_id; - char envconf[512], tableconf[128]; + char envconf[512]; thr = dcalloc(nth + 2, sizeof(*thr)); td = dcalloc(nth + 2, sizeof(THREAD_DATA)); @@ -790,13 +783,10 @@ run_workload(uint32_t nth) /* * Create all the tables. */ - testutil_check(__wt_snprintf(tableconf, sizeof(tableconf), - "key_format=%s,value_format=u,log=(enabled=false)", use_columns ? "r" : "S")); - testutil_check(session->create(session, uri_collection, tableconf)); - testutil_check(__wt_snprintf( - tableconf, sizeof(tableconf), "key_format=%s,value_format=u", use_columns ? "r" : "S")); - testutil_check(session->create(session, uri_local, tableconf)); - testutil_check(session->create(session, uri_oplog, tableconf)); + testutil_check( + session->create(session, uri_collection, "key_format=S,value_format=u,log=(enabled=false)")); + testutil_check(session->create(session, uri_local, "key_format=S,value_format=u")); + testutil_check(session->create(session, uri_oplog, "key_format=S,value_format=u")); /* * Don't log the stable timestamp table so that we know what timestamp was stored at the * checkpoint. @@ -919,15 +909,11 @@ main(int argc, char *argv[]) verify_only = false; working_dir = "WT_TEST.schema-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "Cch:mT:t:vxz")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vxz")) != EOF) switch (ch) { case 'C': compat = true; break; - case 'c': - /* Variable-length columns only; fixed would require considerable changes */ - use_columns = true; - break; case 'h': working_dir = __wt_optarg; break; @@ -1101,16 +1087,10 @@ main(int argc, char *argv[]) key, last_key); break; } - if (use_columns) { - cur_coll->set_key(cur_coll, key + 1); - cur_local->set_key(cur_local, key + 1); - cur_oplog->set_key(cur_oplog, key + 1); - } else { - testutil_check(__wt_snprintf(kname, sizeof(kname), ROW_KEY_FORMAT, key)); - cur_coll->set_key(cur_coll, kname); - cur_local->set_key(cur_local, kname); - cur_oplog->set_key(cur_oplog, kname); - } + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, key)); + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); /* * The collection table should always only have the data as of the checkpoint. */ diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh index e7d21ec30e6..5e82ae180bc 100755 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/smoke.sh @@ -21,14 +21,6 @@ $TEST_WRAPPER $test_bin -t 10 -T 5 $TEST_WRAPPER $test_bin -m -t 10 -T 5 $TEST_WRAPPER $test_bin -C -t 10 -T 5 $TEST_WRAPPER $test_bin -C -m -t 10 -T 5 - -$TEST_WRAPPER $test_bin -c -t 10 -T 5 -$TEST_WRAPPER $test_bin -c -m -t 10 -T 5 -$TEST_WRAPPER $test_bin -c -C -t 10 -T 5 -$TEST_WRAPPER $test_bin -c -C -m -t 10 -T 5 - # FIXME: In WT-6116 the test is failing if timestamps are turned off. #$TEST_WRAPPER $test_bin -m -t 10 -T 5 -z -#$TEST_WRAPPER $test_bin -c -m -t 10 -T 5 -z $TEST_WRAPPER $test_bin -m -t 10 -T 5 -x -$TEST_WRAPPER $test_bin -c -m -t 10 -T 5 -x diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 1d59222104b..6fa41f0d82c 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -79,7 +79,7 @@ static const char *const uri_shadow = "shadow"; static const char *const ckpt_file = "checkpoint_done"; -static bool columns, compat, inmem, stress, use_ts; +static bool compat, inmem, stress, use_ts; static volatile uint64_t global_ts = 1; /* @@ -107,10 +107,9 @@ static volatile uint64_t global_ts = 1; /* * A minimum width of 10, along with zero filling, means that all the keys sort according to their - * integer value, making each thread's key space distinct. For column-store we just use the integer - * values and that has the same effect. + * integer value, making each thread's key space distinct. */ -#define KEY_STRINGFORMAT ("%010" PRIu64) +#define KEY_FORMAT ("%010" PRIu64) typedef struct { uint64_t absent_key; /* Last absent key */ @@ -335,6 +334,8 @@ thread_run(void *arg) printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->info, td->start); active_ts = 0; for (i = td->start;; ++i) { + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, i)); + testutil_check(session->begin_transaction(session, NULL)); if (use_prep) testutil_check(prepared_session->begin_transaction(prepared_session, NULL)); @@ -353,18 +354,10 @@ thread_run(void *arg) testutil_check(pthread_rwlock_unlock(&ts_lock)); } - if (columns) { - cur_coll->set_key(cur_coll, i + 1); - cur_local->set_key(cur_local, i + 1); - cur_oplog->set_key(cur_oplog, i + 1); - cur_shadow->set_key(cur_shadow, i + 1); - } else { - testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_STRINGFORMAT, i)); - cur_coll->set_key(cur_coll, kname); - cur_local->set_key(cur_local, kname); - cur_oplog->set_key(cur_oplog, kname); - cur_shadow->set_key(cur_shadow, kname); - } + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + cur_shadow->set_key(cur_shadow, kname); /* * Put an informative string into the value so that it can be viewed well in a binary dump. */ @@ -466,7 +459,6 @@ run_workload(uint32_t nth) wt_thread_t *thr; uint32_t cache_mb, ckpt_id, i, ts_id; char envconf[512], uri[128]; - const char *table_config, *table_config_nolog; thr = dcalloc(nth + 2, sizeof(*thr)); td = dcalloc(nth + 2, sizeof(THREAD_DATA)); @@ -503,25 +495,19 @@ run_workload(uint32_t nth) printf("wiredtiger_open configuration: %s\n", envconf); testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* * Create all the tables. */ - if (columns) { - table_config_nolog = "key_format=r,value_format=u,log=(enabled=false)"; - table_config = "key_format=r,value_format=u"; - } else { - table_config_nolog = "key_format=S,value_format=u,log=(enabled=false)"; - table_config = "key_format=S,value_format=u"; - } testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); - testutil_check(session->create(session, uri, table_config_nolog)); + testutil_check( + session->create(session, uri, "key_format=S,value_format=u,log=(enabled=false)")); testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_shadow)); - testutil_check(session->create(session, uri, table_config_nolog)); + testutil_check( + session->create(session, uri, "key_format=S,value_format=u,log=(enabled=false)")); testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); - testutil_check(session->create(session, uri, table_config)); + testutil_check(session->create(session, uri, "key_format=S,value_format=u")); testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); - testutil_check(session->create(session, uri, table_config)); + testutil_check(session->create(session, uri, "key_format=S,value_format=u")); /* * Don't log the stable timestamp table so that we know what timestamp was stored at the * checkpoint. @@ -630,7 +616,7 @@ main(int argc, char *argv[]) (void)testutil_set_progname(argv); - columns = compat = inmem = stress = false; + compat = inmem = stress = false; use_ts = true; nth = MIN_TH; rand_th = rand_time = true; @@ -638,15 +624,11 @@ main(int argc, char *argv[]) verify_only = false; working_dir = "WT_TEST.timestamp-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "Cch:LmsT:t:vz")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmsT:t:vz")) != EOF) switch (ch) { case 'C': compat = true; break; - case 'c': - /* Variable-length columns only (for now) */ - columns = true; - break; case 'h': working_dir = __wt_optarg; break; @@ -717,9 +699,9 @@ main(int argc, char *argv[]) compat ? "true" : "false", inmem ? "true" : "false", stress ? "true" : "false", use_ts ? "true" : "false"); printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); - printf("CONFIG: %s%s%s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, - compat ? " -C" : "", columns ? " -c" : "", inmem ? " -m" : "", stress ? " -s" : "", - !use_ts ? " -z" : "", working_dir, nth, timeout); + printf("CONFIG: %s%s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, + compat ? " -C" : "", inmem ? " -m" : "", stress ? " -s" : "", !use_ts ? " -z" : "", + working_dir, nth, timeout); /* * Fork a child to insert as many items. We will then randomly kill the child, run recovery * and make sure all items we wrote exist after recovery runs. @@ -841,20 +823,11 @@ main(int argc, char *argv[]) key, last_key); break; } - - if (columns) { - cur_coll->set_key(cur_coll, key + 1); - cur_local->set_key(cur_local, key + 1); - cur_oplog->set_key(cur_oplog, key + 1); - cur_shadow->set_key(cur_shadow, key + 1); - } else { - testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_STRINGFORMAT, key)); - cur_coll->set_key(cur_coll, kname); - cur_local->set_key(cur_local, kname); - cur_oplog->set_key(cur_oplog, kname); - cur_shadow->set_key(cur_shadow, kname); - } - + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, key)); + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + cur_shadow->set_key(cur_shadow, kname); /* * The collection table should always only have the data as of the checkpoint. The * shadow table should always have the exact same data (or not) as the collection table, diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh index b2c70340f4c..18d7f9b8dae 100755 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/smoke.sh @@ -23,12 +23,8 @@ then fi $TEST_WRAPPER $test_bin $default_test_args -$TEST_WRAPPER $test_bin $default_test_args -c #$TEST_WRAPPER $test_bin $default_test_args -L $TEST_WRAPPER $test_bin -m $default_test_args -$TEST_WRAPPER $test_bin -m $default_test_args -c #$TEST_WRAPPER $test_bin -m $default_test_args -L $TEST_WRAPPER $test_bin -C $default_test_args -$TEST_WRAPPER $test_bin -C $default_test_args -c $TEST_WRAPPER $test_bin -C -m $default_test_args -$TEST_WRAPPER $test_bin -C -m $default_test_args -c diff --git a/src/third_party/wiredtiger/test/csuite/truncated_log/main.c b/src/third_party/wiredtiger/test/csuite/truncated_log/main.c index fa45e573781..d46b75d48c9 100644 --- a/src/third_party/wiredtiger/test/csuite/truncated_log/main.c +++ b/src/third_party/wiredtiger/test/csuite/truncated_log/main.c @@ -32,7 +32,6 @@ static char home[1024]; /* Program working dir */ static const char *const uri = "table:main"; -static bool use_columns = false; #define RECORDS_FILE "records" @@ -129,14 +128,8 @@ fill_db(void) WT_SESSION *session; uint32_t i, max_key, min_key, units, unused; char k[K_SIZE], v[V_SIZE]; - const char *table_config; bool first; - if (use_columns) - table_config = "key_format=r,value_format=S"; - else - table_config = "key_format=S,value_format=S"; - /* * Run in the home directory so that the records file is in there too. */ @@ -144,7 +137,7 @@ fill_db(void) testutil_die(errno, "chdir: %s", home); testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->create(session, uri, table_config)); + testutil_check(session->create(session, uri, "key_format=S,value_format=S")); testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); /* @@ -171,14 +164,10 @@ fill_db(void) max_key = min_key * 2; first = true; for (i = 0; i < max_key; ++i) { - if (use_columns) - cursor->set_key(cursor, i + 1); - else { - testutil_check(__wt_snprintf(k, sizeof(k), "key%03" PRIu32, i)); - cursor->set_key(cursor, k); - } + testutil_check(__wt_snprintf(k, sizeof(k), "key%03d", (int)i)); testutil_check( - __wt_snprintf(v, sizeof(v), "value%0*" PRIu32, (int)(V_SIZE - (strlen("value") + 1)), i)); + __wt_snprintf(v, sizeof(v), "value%0*d", (int)(V_SIZE - (strlen("value") + 1)), (int)i)); + cursor->set_key(cursor, k); cursor->set_value(cursor, v); testutil_check(cursor->insert(cursor)); @@ -241,12 +230,8 @@ main(int argc, char *argv[]) (void)testutil_set_progname(argv); working_dir = "WT_TEST.truncated-log"; - while ((ch = __wt_getopt(progname, argc, argv, "ch:")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "h:")) != EOF) switch (ch) { - case 'c': - /* Variable-length columns only (for now) */ - use_columns = true; - break; case 'h': working_dir = __wt_optarg; break; diff --git a/src/third_party/wiredtiger/test/csuite/truncated_log/smoke.sh b/src/third_party/wiredtiger/test/csuite/truncated_log/smoke.sh deleted file mode 100755 index 0079adf0340..00000000000 --- a/src/third_party/wiredtiger/test/csuite/truncated_log/smoke.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /bin/sh - -set -e - -# Smoke-test truncated_log as part of running "make check". - -if [ -n "$1" ] -then - # If the test binary is passed in manually. - test_bin=$1 -else - # If $top_builddir/$top_srcdir aren't set, default to building in build_posix - # and running in test/csuite. - top_builddir=${top_builddir:-../../build_posix} - top_srcdir=${top_srcdir:-../..} - test_bin=$top_builddir/test/csuite/test_truncated_log -fi - -$TEST_WRAPPER $test_bin -$TEST_WRAPPER $test_bin -c diff --git a/src/third_party/wiredtiger/test/csuite/wt6185_modify_ts/main.c b/src/third_party/wiredtiger/test/csuite/wt6185_modify_ts/main.c index 02205c88429..1f2e824047b 100644 --- a/src/third_party/wiredtiger/test/csuite/wt6185_modify_ts/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt6185_modify_ts/main.c @@ -50,10 +50,7 @@ static u_int tnext; static uint64_t ts; /* Current timestamp. */ -static char keystr[100], modify_repl[256], tmp[4 * 1024]; -static uint64_t keyrecno; - -static bool use_columns = false; +static char key[100], modify_repl[256], tmp[4 * 1024]; /* * trace -- @@ -119,32 +116,6 @@ mmrand(u_int min, u_int max) return (v); } -/* - * change_key -- - * Switch to a different key. - */ -static void -change_key(u_int n) -{ - if (use_columns) - keyrecno = n + 1; - else - testutil_check(__wt_snprintf(keystr, sizeof(keystr), "%010u.key", n)); -} - -/* - * set_key -- - * Set the current key in the cursor. - */ -static void -set_key(WT_CURSOR *c) -{ - if (use_columns) - c->set_key(c, keyrecno); - else - c->set_key(c, keystr); -} - /* * modify_repl_init -- * Initialize the replacement information. @@ -210,13 +181,13 @@ modify(WT_SESSION *session, WT_CURSOR *c) for (cnt = loop = 1; loop < 5; ++cnt, ++loop) if (mmrand(1, 10) <= 8) { modify_build(entries, &nentries, cnt); - set_key(c); + c->set_key(c, key); testutil_check(c->modify(c, entries, nentries)); } /* Commit 90% of the time, else rollback. */ if (mmrand(1, 10) != 1) { - set_key(c); + c->set_key(c, key); testutil_check(c->search(c)); testutil_check(c->get_value(c, &v)); free(list[lnext].v); @@ -252,7 +223,7 @@ repeat(WT_SESSION *session, WT_CURSOR *c) testutil_check(__wt_snprintf(tmp, sizeof(tmp), "read_timestamp=%" PRIx64, list[i].ts)); testutil_check(session->timestamp_transaction(session, tmp)); - set_key(c); + c->set_key(c, key); testutil_check(c->search(c)); testutil_check(c->get_value(c, &v)); @@ -275,7 +246,7 @@ evict(WT_CURSOR *c) { trace("%s", "eviction"); - set_key(c); + c->set_key(c, key); testutil_check(c->search(c)); F_SET(c, WT_CURSTD_DEBUG_RESET_EVICT); testutil_check(c->reset(c)); @@ -315,7 +286,7 @@ main(int argc, char *argv[]) WT_SESSION *session; u_int i, j; int ch; - char path[1024], table_config[128], value[VALUE_SIZE]; + char path[1024], value[VALUE_SIZE]; const char *home, *v; bool no_checkpoint, no_eviction; @@ -327,12 +298,8 @@ main(int argc, char *argv[]) no_checkpoint = no_eviction = false; home = "WT_TEST.wt6185_modify_ts"; - while ((ch = __wt_getopt(progname, argc, argv, "Cceh:S:")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "ceh:S:")) != EOF) switch (ch) { - case 'C': - /* Variable-length columns only (for now anyway) */ - use_columns = true; - break; case 'c': no_checkpoint = true; break; @@ -355,17 +322,14 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(path, sizeof(path), home); testutil_make_work_dir(path); - testutil_check(__wt_snprintf( - table_config, sizeof(table_config), "key_format=%s,value_format=S", use_columns ? "r" : "S")); - /* Load 100 records. */ testutil_check(wiredtiger_open(path, NULL, "create", &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->create(session, "file:xxx", table_config)); + testutil_check(session->create(session, "file:xxx", "key_format=S,value_format=S")); testutil_check(session->open_cursor(session, "file:xxx", NULL, NULL, &c)); for (i = 0; i <= 100; ++i) { - change_key(i); - set_key(c); + testutil_check(__wt_snprintf(key, sizeof(key), "%010u.key", i)); + c->set_key(c, key); SET_VALUE(i, value); c->set_value(c, value); testutil_check(c->insert(c)); @@ -377,8 +341,8 @@ main(int argc, char *argv[]) testutil_check(conn->open_session(conn, NULL, NULL, &session)); testutil_check(session->create(session, "file:xxx", NULL)); testutil_check(session->open_cursor(session, "file:xxx", NULL, NULL, &c)); - change_key(KEYNO); - set_key(c); + testutil_check(__wt_snprintf(key, sizeof(key), "%010d.key", KEYNO)); + c->set_key(c, key); testutil_check(c->search(c)); testutil_check(c->get_value(c, &v)); SET_VALUE(KEYNO, value); diff --git a/src/third_party/wiredtiger/test/csuite/wt6185_modify_ts/smoke.sh b/src/third_party/wiredtiger/test/csuite/wt6185_modify_ts/smoke.sh deleted file mode 100755 index b317eeeb2ed..00000000000 --- a/src/third_party/wiredtiger/test/csuite/wt6185_modify_ts/smoke.sh +++ /dev/null @@ -1,21 +0,0 @@ -#! /bin/sh - -set -e - -# Smoke-test wt6185_modify_ts as part of running "make check". - -if [ -n "$1" ] -then - # If the test binary is passed in manually. - test_bin=$1 -else - # If $top_builddir/$top_srcdir aren't set, default to building in build_posix - # and running in test/csuite. - top_builddir=${top_builddir:-../../build_posix} - top_srcdir=${top_srcdir:-../..} - test_bin=$top_builddir/test/csuite/test_wt6185_modify_ts -fi - -$TEST_WRAPPER $test_bin -$TEST_WRAPPER $test_bin -C - diff --git a/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c b/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c index 2e9648efea0..5a75777aa78 100644 --- a/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/main.c @@ -32,7 +32,6 @@ #include static char home[1024]; /* Program working dir */ -static bool use_columns = false; /* * Spin up a child process to do operations and checkpoint. For each set of operations on a key, @@ -49,7 +48,7 @@ static bool use_columns = false; * recovery by reading without a timestamp. Whether it is possible to read historical versions based * on timestamps from a logged table after recovery is not defined and implemented yet. */ -#define ROW_KEY_FORMAT ("%010" PRIu64) +#define KEY_FORMAT ("%010" PRIu64) #define MAX_CKPT_INVL 5 /* Maximum interval between checkpoints */ #define MAX_DATA 1000 @@ -148,14 +147,11 @@ thread_run(void *arg) /* Insert and then delete the keys until we're killed. */ printf("Worker thread started.\n"); for (oldest_ts = 0, ts = 1;; ++ts) { - testutil_check(__wt_snprintf(kname, sizeof(kname), ROW_KEY_FORMAT, ts)); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, ts)); /* Insert the same value for key and value. */ testutil_check(session->begin_transaction(session, NULL)); - if (use_columns) - cursor->set_key(cursor, ts); - else - cursor->set_key(cursor, kname); + cursor->set_key(cursor, kname); data.data = kname; data.size = sizeof(kname); cursor->set_value(cursor, &data); @@ -197,7 +193,7 @@ run_workload(void) WT_SESSION *session; wt_thread_t *thr; uint32_t i; - char envconf[512], tableconf[512]; + char envconf[512]; thr = dcalloc(2, sizeof(*thr)); @@ -210,9 +206,8 @@ run_workload(void) testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* Create the table. */ - testutil_check(__wt_snprintf(tableconf, sizeof(tableconf), - "key_format=%s,value_format=u,log=(enabled=false)", use_columns ? "r" : "S")); - testutil_check(session->create(session, uri, tableconf)); + testutil_check( + session->create(session, uri, "key_format=S,value_format=u,log=(enabled=false)")); testutil_check(session->close(session, NULL)); /* The checkpoint thread is added at the end. */ @@ -273,12 +268,8 @@ main(int argc, char *argv[]) timeout = MIN_TIME; working_dir = "WT_TEST.wt6616-checkpoint-oldest-ts"; - while ((ch = __wt_getopt(progname, argc, argv, "ch:t:")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "h:t:")) != EOF) switch (ch) { - case 'c': - /* Variable-length columns only (for now) */ - use_columns = true; - break; case 'h': working_dir = __wt_optarg; break; @@ -372,11 +363,8 @@ main(int argc, char *argv[]) for (ts = oldest_ts; ts <= stable_ts; ++ts) { testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), "read_timestamp=%" PRIx64, ts)); testutil_check(session->begin_transaction(session, tscfg)); - testutil_check(__wt_snprintf(kname, sizeof(kname), ROW_KEY_FORMAT, ts)); - if (use_columns) - cursor->set_key(cursor, ts); - else - cursor->set_key(cursor, kname); + testutil_check(__wt_snprintf(kname, sizeof(kname), KEY_FORMAT, ts)); + cursor->set_key(cursor, kname); ret = cursor->search(cursor); if (ret == WT_NOTFOUND) { fatal = true; diff --git a/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/smoke.sh b/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/smoke.sh deleted file mode 100755 index 9b9cc997026..00000000000 --- a/src/third_party/wiredtiger/test/csuite/wt6616_checkpoint_oldest_ts/smoke.sh +++ /dev/null @@ -1,21 +0,0 @@ -#! /bin/sh - -set -e - -# Smoke-test wt6616_checkpoint_oldest_ts as part of running "make check". - -if [ -n "$1" ] -then - # If the test binary is passed in manually. - test_bin=$1 -else - # If $top_builddir/$top_srcdir aren't set, default to building in build_posix - # and running in test/csuite. - top_builddir=${top_builddir:-../../build_posix} - top_srcdir=${top_srcdir:-../..} - test_bin=$top_builddir/test/csuite/test_wt6616_checkpoint_oldest_ts -fi - -$TEST_WRAPPER $test_bin -$TEST_WRAPPER $test_bin -c - diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index ce20b4ac2b5..5bc5fa6580b 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -2580,17 +2580,6 @@ tasks: name: recovery-stress-test-3 tags: ["stress-test-3", "stress-test-zseries-3"] - - name: format-abort-recovery-stress-test - commands: - - command: timeout.update - params: - exec_timeout_secs: 2500 - - func: "get project" - - func: "compile wiredtiger with builtins" - - func: "format test script" - vars: - format_test_script_args: -a -t 30 - - name: many-dhandle-stress-test commands: - func: "get project" @@ -2841,7 +2830,6 @@ buildvariants: - name: ".stress-test-2" - name: ".stress-test-3" - name: ".stress-test-4" - - name: format-abort-recovery-stress-test - name: large-scale-tests display_name: "Large scale tests" diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 33b8d7a9112..88964902d98 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -715,8 +715,6 @@ config_in_memory(void) return; if (config_is_perm("checkpoint")) return; - if (config_is_perm("format.abort")) - return; if (config_is_perm("import")) return; if (config_is_perm("logging")) diff --git a/src/third_party/wiredtiger/test/suite/test_hs18.py b/src/third_party/wiredtiger/test/suite/test_hs18.py index 5ed21e3c90a..bcef53e4d17 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs18.py +++ b/src/third_party/wiredtiger/test/suite/test_hs18.py @@ -438,6 +438,8 @@ class test_hs18(wttest.WiredTigerTestCase): session_ts_reader = self.setUpSessionOpen(self.conn) cursor_ts_reader = session_ts_reader.open_cursor(uri) + self.skipTest('Skip this part of test_hs18 until WT-7931 is resolved') + # The ID of the session corresponds the value it should see. sessions = [] cursors = [] @@ -446,6 +448,8 @@ class test_hs18(wttest.WiredTigerTestCase): sessions.append(self.setUpSessionOpen(self.conn)) cursors.append(sessions[i].open_cursor(uri)) + value_junk = 'aaaaa' * 100 + values.append('f' * 10) values.append('a' + values[0]) values.append('b' + values[1]) @@ -481,13 +485,11 @@ class test_hs18(wttest.WiredTigerTestCase): # Start a long running transaction which could see modify 1. self.start_txn(sessions, cursors, values, 2) - # Evict the update using a debug cursor - cursor.reset() - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") - evict_cursor.set_key(self.create_key(1)) - self.assertEqual(evict_cursor.search(), 0) - evict_cursor.reset() - evict_cursor.close() + # Insert a bunch of contents to fill the cache + for i in range(2000, 10000): + self.session.begin_transaction() + cursor[self.create_key(i)] = value_junk + self.session.commit_transaction() # Commit a modify without a timestamp on our original key self.session.begin_transaction() @@ -509,13 +511,11 @@ class test_hs18(wttest.WiredTigerTestCase): for i in range(0, 5): self.check_value(cursors[i], values[i]) - # Evict the update using a debug cursor - cursor.reset() - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") - evict_cursor.set_key(self.create_key(1)) - self.assertEqual(evict_cursor.search(), 0) - evict_cursor.reset() - evict_cursor.close() + # Insert a bunch of other contents to trigger eviction + for i in range(10001, 11000): + self.session.begin_transaction() + cursor[self.create_key(i)] = value_junk + self.session.commit_transaction() # Check our values are still correct. for i in range(0, 5): diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py index 7c85800b070..f6ef52cc388 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py @@ -43,6 +43,8 @@ class test_rollback_to_stable22(test_rollback_to_stable_base): nrows = 1000 nds = 10 + self.skipTest('Skip it until the fix is provided to handle concurrent internal transactions running in parallel.') + # Create a few tables and populate them with some initial data. # # Our way of preventing history store operations from interfering with rollback to stable's diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable24.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable24.py deleted file mode 100644 index ea690506dc9..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable24.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios - -# test_rollback_to_stable24.py -# Exercise a recno-counting bug in column store. -# -# Prior to August 2021 a cell for which there's a pending stable update was counted (in the -# column-store RTS code) as having RLE count 1 regardless of what the actual count was. -# -# In order to exploit this we have to do janky things with timestamps, but I think they're -# allowable. -# -# Construct a cell with RLE count of 3 by writing 3 copies of aaaaaa at timestamp 10. -# Then at the next key write bbbbbb at timestamp 10 and cccccc at timestamp 50. -# Evict the page to reconcile it and produce the RLE cell. -# -# Then post an update to the first key of the RLE cell at timestamp 30 (to dddddd), and roll -# back to 40. -# -# Reading at 40, we should at that point see dddddd and two aaaaaa's followed by bbbbbb, but -# with the bad counting we get a key error on the second key. -# -# This happens because it goes to process key 4 but thinks it's on key 2; it finds that it -# needs to roll back the value it's looking at (the cccccc from timestamp 50) but because it -# thinks it's on key to it asks the history store for key 2 and finds nothing. (The bbbbbb -# from timestamp 10 is in the history store, but under key 4; there's nothing in the history -# store for key 2.) So it issues a tombstone, and issues it for key 2, so key 2 improperly -# disappears. -# -# Run this test on rows as well as columns to help make sure the test itself is valid (and -# stays so over time...) -class test_rollback_to_stable24(wttest.WiredTigerTestCase): - session_config = 'isolation=snapshot' - conn_config = 'in_memory=false' - - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), - ] - - scenarios = make_scenarios(key_format_values) - - def test_rollback_to_stable24(self): - # Create a table without logging. - uri = "table:rollback_to_stable24" - format = 'key_format={},value_format=S'.format(self.key_format) - self.session.create(uri, format + ', log=(enabled=false)') - - # Pin oldest timestamp to 10. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) - - # Start stable timestamp at 10. - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) - - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - - s = self.conn.open_session() - cursor = s.open_cursor(uri) - - # Write some keys at time 10. - s.begin_transaction() - cursor[1] = value_a - cursor[2] = value_a - cursor[3] = value_a - cursor[4] = value_b - s.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) - - # Update key 4 at time 50. - s.begin_transaction() - cursor[4] = value_c - s.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) - - cursor.close() - - # Evict the page to force reconciliation. - evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") - s.begin_transaction() - # Search the key to evict it. - v = evict_cursor[1] - self.assertEqual(v, value_a) - self.assertEqual(evict_cursor.reset(), 0) - s.rollback_transaction() - evict_cursor.close() - - # Now update key 1 at time 30. - cursor = s.open_cursor(uri) - s.begin_transaction() - cursor[1] = value_d - s.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) - cursor.close() - - # Roll back to 40. - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) - self.conn.rollback_to_stable() - - # Now read at 40. - cursor = s.open_cursor(uri) - s.begin_transaction('read_timestamp=' + self.timestamp_str(40)) - self.assertEqual(cursor[1], value_d) - self.assertEqual(cursor[2], value_a) - self.assertEqual(cursor[3], value_a) - self.assertEqual(cursor[4], value_b) - s.rollback_transaction() - cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable25.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable25.py deleted file mode 100644 index 2d800a17d32..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable25.py +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -import wiredtiger, wttest -from wtscenario import make_scenarios, filter_scenarios - -# test_rollback_to_stable25.py -# Check various scenarios relating to RLE cells in column-store. -# -# We write at three different timestamps: -# 10 - aaaaaa or none -# 20 - bbbbbb or delete or none -# 30 - cccccc or delete or none -# -# and we evict to push things to disk after any of these, -# and we roll back to either 15 or 25. -# -# The writes can be either uniform, heterogeneous, first key, middle key, or last key. -# -# We do this with a group of 5 keys 2..6. Keys 1 and 6 are written with zzzzzz at -# timestamp 5 and evicted to ensure that the group of keys we're using is isolated -# from other unused keys. -# -# This generates a lot of cases, but we filter pointless combinations and they run fast. - -# Put these bits outside the class definition so they can be referred to both in class -# instances and in the scenario setup logic, which doesn't have a class instance yet. - -my_rle_size = 5 - -def keys_of_write(write): - if write == 'u' or write == 'h': - return range(2, 2 + my_rle_size) - elif write == 'f': - return [2] - elif write == 'm': - return [2 + my_rle_size // 2] - else: - return [2 + my_rle_size - 1] - -class test_rollback_to_stable25(wttest.WiredTigerTestCase): - session_config = 'isolation=snapshot' - conn_config = 'in_memory=false' - - write_10_values = [ - ('10u', dict(write_10='u')), - ('10h', dict(write_10='h')), - ('10f', dict(write_10='f')), - ('10m', dict(write_10='m')), - ('10l', dict(write_10='l')), - ] - type_10_values = [ - ('nil', dict(type_10=None)), - ('upd', dict(type_10='upd')), - ] - - write_20_values = [ - ('20u', dict(write_20='u')), - ('20h', dict(write_20='h')), - ('20f', dict(write_20='f')), - ('20m', dict(write_20='m')), - ('20l', dict(write_20='l')), - ] - type_20_values = [ - ('nil', dict(type_20=None)), - ('upd', dict(type_20='upd')), - ('del', dict(type_20='del')), - ] - - write_30_values = [ - ('30u', dict(write_30='u')), - ('30h', dict(write_30='h')), - ('30f', dict(write_30='f')), - ('30m', dict(write_30='m')), - ('30l', dict(write_30='l')), - ] - type_30_values = [ - ('nil', dict(type_30=None)), - ('upd', dict(type_30='upd')), - ('del', dict(type_30='del')), - ] - - evict_time_values = [ - ('chk10', dict(evict_time=10)), - ('chk20', dict(evict_time=20)), - ('chk30', dict(evict_time=30)), - ] - - rollback_time_values = [ - ('roll15', dict(rollback_time=15)), - ('roll25', dict(rollback_time=25)), - ] - - def is_meaningful(name, vals): - # The last write at evict time should be uniform, to get an RLE cell. - if vals['evict_time'] == 10 and vals['write_10'] != 'u': - return False - if vals['evict_time'] == 20 and vals['write_20'] != 'u': - return False - if vals['evict_time'] == 30 and vals['write_30'] != 'u': - return False - # If the type is nil, the value must be uniform. - if vals['type_10'] is None and vals['write_10'] != 'u': - return False - if vals['type_20'] is None and vals['write_20'] != 'u': - return False - if vals['type_30'] is None and vals['write_30'] != 'u': - return False - # Similarly, delete and heterogeneous doesn't make sense. - if vals['type_10'] == 'del' and vals['write_10'] == 'h': - return False - if vals['type_20'] == 'del' and vals['write_20'] == 'h': - return False - if vals['type_20'] == 'del' and vals['write_30'] == 'h': - return False - # Both 10 and 20 shouldn't be nil. That's equivalent to 10 and 30 being nil. - if vals['type_10'] is None and vals['type_20'] is None: - return False - - # Avoid cases that delete nonexistent values. - def deletes_nonexistent(): - present = {} - for k in range(2, 2 + my_rle_size): - present[k] = False - def adjust(ty, write): - if ty is None: - return - for k in keys_of_write(write): - if ty == 'upd': - present[k] = True - elif ty == 'del': - if present[k]: - present[k] = False - else: - raise KeyError - - adjust(vals['type_10'], vals['write_10']) - adjust(vals['type_20'], vals['write_20']) - adjust(vals['type_30'], vals['write_30']) - try: - deletes_nonexistent() - except KeyError: - return False - return True - - scenarios = filter_scenarios(make_scenarios(write_10_values, type_10_values, - write_20_values, type_20_values, - write_30_values, type_30_values, - evict_time_values, - rollback_time_values), - is_meaningful) - - value_z = "zzzzz" * 10 - - def writes(self, uri, s, expected, ty, write, value, ts): - if ty is None: - # do nothing at all - return - cursor = s.open_cursor(uri) - s.begin_transaction() - for k in keys_of_write(write): - if ty == 'upd': - myval = value + str(k) if write == 'h' else value - cursor[k] = myval - expected[k] = myval - else: - cursor.set_key(k) - cursor.remove() - del expected[k] - s.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) - cursor.close() - - def evict(self, uri, s): - # Evict the page to force reconciliation. - evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") - s.begin_transaction() - # Search the key to evict it. Use both bookends. - v = evict_cursor[1] - self.assertEqual(v, self. value_z) - v = evict_cursor[2 + my_rle_size] - self.assertEqual(v, self. value_z) - self.assertEqual(evict_cursor.reset(), 0) - s.rollback_transaction() - evict_cursor.close() - - def check(self, uri, s, ts, expected): - cursor = s.open_cursor(uri) - s.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) - # endpoints should still be in place - self.assertEqual(cursor[1], self.value_z) - self.assertEqual(cursor[2 + my_rle_size], self.value_z) - - for k in range(2, 2 + my_rle_size): - if k in expected: - self.assertEqual(cursor[k], expected[k]) - else: - cursor.set_key(k) - r = cursor.search() - self.assertEqual(r, wiredtiger.WT_NOTFOUND) - s.rollback_transaction() - cursor.close() - - def test_rollback_to_stable25(self): - # Create a table without logging. - uri = "table:rollback_to_stable25" - format = 'key_format=r,value_format=S' - self.session.create(uri, format + ', log=(enabled=false)') - - # Pin oldest timestamp to 5. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) - - # Start stable timestamp at 5. - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) - - value_a = "aaaaa" * 10 - value_b = "bbbbb" * 10 - value_c = "ccccc" * 10 - - s = self.conn.open_session() - - # Write the endpoints at time 5. - cursor = s.open_cursor(uri) - s.begin_transaction() - cursor[1] = self.value_z - cursor[2 + my_rle_size] = self.value_z - s.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) - self.evict(uri, s) - cursor.close() - - # Do writes at time 10. - expected = {} - self.writes(uri, s, expected, self.type_10, self.write_10, value_a, 10) - expected10 = expected.copy() - - # Evict at time 10 if requested. - if self.evict_time == 10: - self.evict(uri, s) - - # Do more writes at time 20. - self.writes(uri, s, expected, self.type_20, self.write_20, value_b, 20) - expected20 = expected.copy() - - # Evict at time 20 if requested. - if self.evict_time == 20: - self.evict(uri, s) - - # Do still more writes at time 30. - self.writes(uri, s, expected, self.type_30, self.write_30, value_c, 30) - expected30 = expected.copy() - - # Evict at time 30 if requested. - if self.evict_time == 30: - self.evict(uri, s) - - # Now roll back. - self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.rollback_time)) - self.conn.rollback_to_stable() - - if self.rollback_time < 20: - expected20 = expected10 - expected30 = expected10 - elif self.rollback_time < 30: - expected30 = expected20 - - # Now make sure we see what we expect. - self.check(uri, s, 10, expected10) - self.check(uri, s, 20, expected20) - self.check(uri, s, 30, expected30) -- cgit v1.2.1