diff options
author | Luke Chen <luke.chen@mongodb.com> | 2021-08-16 14:53:17 +1000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-08-16 05:21:34 +0000 |
commit | e00b31ac8a5509ebca265178daa3a1d1e2ab3bb2 (patch) | |
tree | ef6a7224febfdc095e6f7251d605f3a373b7639b /src/third_party | |
parent | 5a20195efb81fa18645498eac5a0865372b8a663 (diff) | |
download | mongo-e00b31ac8a5509ebca265178daa3a1d1e2ab3bb2.tar.gz |
Import wiredtiger: 4fed751669a4ca66b882f73a8e8d555174a4da66 from branch mongodb-master
ref: 8099896388..4fed751669
for: 5.1.0
WT-7928 VLCS checkpoint and additional test suite improvements
Diffstat (limited to 'src/third_party')
28 files changed, 398 insertions, 157 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index e48dcc7aa78..83ea8da2184 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "8099896388aa3476a53d36698018fc69e4240ab5" + "commit": "4fed751669a4ca66b882f73a8e8d555174a4da66" } diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index ef8e42f97e5..c952272bb8b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -944,7 +944,7 @@ __curfile_update_check(WT_CURSOR_BTREE *cbt) else if (btree->type != BTREE_COL_VAR) return (0); - return (__wt_txn_update_check(session, cbt, upd, NULL)); + return (__wt_txn_modify_check(session, cbt, upd, NULL)); } /* diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index 632526f83df..c14d3251042 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -35,15 +35,16 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; u_int i, skipdepth; - bool append, logged; + bool append, inserted_to_update_chain, logged; btree = CUR2BT(cbt); ins = NULL; page = cbt->ref->page; session = CUR2S(cbt); + last_upd = NULL; upd = upd_arg; prev_upd_ts = WT_TS_NONE; - append = logged = false; + append = inserted_to_update_chain = logged = false; /* * We should have one of the following: @@ -120,19 +121,15 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U } /* - * Delete, insert or update a column-store entry. - * - * If modifying a previously modified record, cursor.ins will be set to point to the correct - * update list. Create a new update entry and link it into the existing list. - * - * Else, allocate an insert array as necessary, build an insert/update structure pair, and link - * it into place. + * Modify a column-store entry. If modifying a previously modified record, cursor.ins will point + * to the correct update list; create a new update and link it into the already existing list. + * Otherwise, we have to insert a new insert/update pair into the column-store insert list. */ if (cbt->compare == 0 && cbt->ins != NULL) { old_upd = cbt->ins->upd; if (upd_arg == NULL) { - /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd, &prev_upd_ts)); + /* Make sure the modify can proceed. */ + WT_ERR(__wt_txn_modify_check(session, cbt, old_upd, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); @@ -142,7 +139,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_ERR(__wt_txn_modify(session, upd)); logged = true; - /* Avoid a data copy in WT_CURSOR.update. */ + /* Avoid WT_CURSOR.update data copy. */ __wt_upd_value_assign(cbt->modify_update, upd); } else { upd_size = __wt_update_list_memsize(upd); @@ -178,6 +175,10 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U /* Serialize the update. */ WT_ERR(__wt_update_serial(session, cbt, page, &cbt->ins->upd, &upd, upd_size, false)); } else { + /* Make sure the modify can proceed. */ + if (cbt->compare == 0 && upd_arg == NULL) + WT_ERR(__wt_txn_modify_check(session, cbt, NULL, NULL)); + /* Allocate the append/update list reference as necessary. */ if (append) { WT_PAGE_ALLOC_AND_SWAP(session, page, mod->mod_col_append, ins_headp, 1); @@ -218,7 +219,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_ERR(__wt_txn_modify(session, upd)); logged = true; - /* Avoid a data copy in WT_CURSOR.update. */ + /* Avoid WT_CURSOR.update data copy. */ __wt_upd_value_assign(cbt->modify_update, upd); } else upd_size = __wt_update_list_memsize(upd); @@ -254,12 +255,14 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U session, page, cbt->ins_head, cbt->ins_stack, &ins, ins_size, skipdepth, exclusive)); } + inserted_to_update_chain = true; + /* If the update was successful, add it to the in-memory log. */ if (logged && modify_type != WT_UPDATE_RESERVE) { WT_ERR(__wt_txn_log_op(session, cbt)); /* - * In case of append, the recno (key) for the value is assigned now. Set the recno in the + * In case of append, the recno (key) for the value is assigned now. Set the key in the * transaction operation to be used in case this transaction is prepared to retrieve the * update corresponding to this operation. */ @@ -268,14 +271,25 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U if (0) { err: - /* - * Remove the update from the current transaction, so we don't try to modify it on rollback. - */ + /* Remove the update from the current transaction; don't try to modify it on rollback. */ if (logged) __wt_txn_unmodify(session); + + /* Free any allocated insert list object. */ __wt_free(session, ins); - if (upd_arg == NULL) + + cbt->ins = NULL; + + /* Discard any allocated update, unless we failed after linking it into page memory. */ + if (upd_arg == NULL && !inserted_to_update_chain) __wt_free(session, upd); + + /* + * When prepending a list of updates to an update chain, we link them together; sever that + * link so our callers list doesn't point into page memory. + */ + if (last_upd != NULL) + last_upd->next = NULL; } return (ret); diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index c96b963cb1f..779f5112c12 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -109,8 +109,8 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, upd_entry = &cbt->ins->upd; if (upd_arg == NULL) { - /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd = *upd_entry, &prev_upd_ts)); + /* Make sure the modify can proceed. */ + WT_ERR(__wt_txn_modify_check(session, cbt, old_upd = *upd_entry, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); @@ -139,6 +139,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, upd_arg->next == NULL) || (upd_arg->type == WT_UPDATE_TOMBSTONE && upd_arg->next != NULL && upd_arg->next->type == WT_UPDATE_STANDARD && upd_arg->next->next == NULL)); + upd_size = __wt_update_list_memsize(upd); /* If there are existing updates, append them after the new updates. */ @@ -182,7 +183,6 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, * slot. That's hard, so we set a flag. */ WT_PAGE_ALLOC_AND_SWAP(session, page, mod->mod_row_insert, ins_headp, page->entries + 1); - ins_slot = F_ISSET(cbt, WT_CBT_SEARCH_SMALLEST) ? page->entries : cbt->slot; ins_headp = &mod->mod_row_insert[ins_slot]; @@ -218,6 +218,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, (upd_arg->type == WT_UPDATE_TOMBSTONE && upd_arg->next != NULL && upd_arg->next->type == WT_UPDATE_STANDARD && upd_arg->next->next == NULL) || (upd_arg->type == WT_UPDATE_STANDARD && upd_arg->next == NULL)); + upd_size = __wt_update_list_memsize(upd); } @@ -250,8 +251,10 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, inserted_to_update_chain = true; + /* If the update was successful, add it to the in-memory log. */ if (logged && modify_type != WT_UPDATE_RESERVE) { WT_ERR(__wt_txn_log_op(session, cbt)); + /* * Set the key in the transaction operation to be used in case this transaction is prepared * to retrieve the update corresponding to this operation. @@ -261,15 +264,23 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, if (0) { err: - /* - * Remove the update from the current transaction, so we don't try to modify it on rollback. - */ + /* Remove the update from the current transaction, don't try to modify it on rollback. */ if (logged) __wt_txn_unmodify(session); + + /* Free any allocated insert list object. */ __wt_free(session, ins); + cbt->ins = NULL; + + /* Discard any allocated update, unless we failed after linking it into page memory. */ if (upd_arg == NULL && !inserted_to_update_chain) __wt_free(session, upd); + + /* + * When prepending a list of updates to an update chain, we link them together; sever that + * link so our callers list doesn't point into page memory. + */ if (last_upd != NULL) last_upd->next = NULL; } diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 68b89d99b0e..f22d3ddfd0f 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -2134,6 +2134,8 @@ static inline int __wt_txn_idle_cache_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_txn_modify_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, + WT_UPDATE *upd, wt_timestamp_t *prev_tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_modify_page_delete(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_op_set_key(WT_SESSION_IMPL *session, const WT_ITEM *key) @@ -2147,8 +2149,6 @@ static inline int __wt_txn_read_upd_list_internal(WT_SESSION_IMPL *session, WT_C WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_search_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_UPDATE *upd, wt_timestamp_t *prev_tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, u_int modify_type, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp, diff --git a/src/third_party/wiredtiger/src/include/serial_inline.h b/src/third_party/wiredtiger/src/include/serial_inline.h index 8e6813f2866..4f6384732dc 100644 --- a/src/third_party/wiredtiger/src/include/serial_inline.h +++ b/src/third_party/wiredtiger/src/include/serial_inline.h @@ -242,7 +242,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page * Check if our update is still permitted. */ while (!__wt_atomic_cas_ptr(srch_upd, upd->next, upd)) { - if ((ret = __wt_txn_update_check(session, cbt, upd->next = *srch_upd, &prev_upd_ts)) != 0) { + if ((ret = __wt_txn_modify_check(session, cbt, upd->next = *srch_upd, &prev_upd_ts)) != 0) { /* Free unused memory on error. */ __wt_free(session, upd); return (ret); diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index df81b29fe23..6d3b4e8fa55 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -1304,11 +1304,11 @@ __wt_txn_search_check(WT_SESSION_IMPL *session) } /* - * __wt_txn_update_check -- - * Check if the current transaction can update an item. + * __wt_txn_modify_check -- + * Check if the current transaction can modify an item. */ static inline int -__wt_txn_update_check( +__wt_txn_modify_check( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, wt_timestamp_t *prev_tsp) { WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 9518a3b63c3..4ea57b8acc7 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -327,7 +327,9 @@ __wt_rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) WT_RET(__wt_rec_split_init(session, r, page, pageref->ref_recno, btree->maxleafpage)); /* Copy the original, disk-image bytes into place. */ - memcpy(r->first_free, page->pg_fix_bitf, __bitstr_size((size_t)page->entries * btree->bitcnt)); + if (page->entries != 0) + memcpy( + r->first_free, page->pg_fix_bitf, __bitstr_size((size_t)page->entries * btree->bitcnt)); /* Update any changes to the original on-page data items. */ WT_SKIP_FOREACH (ins, WT_COL_UPDATE_SINGLE(page)) { diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 3bb80e94156..b7e7a9a952b 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -82,7 +82,6 @@ clock_thread(void *arg) testutil_check(g.conn->open_session(g.conn, NULL, NULL, &wt_session)); session = (WT_SESSION_IMPL *)wt_session; - g.ts_stable = 0; while (g.running) { __wt_writelock(session, &g.clock_lock); ++g.ts_stable; diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 83f1b1f6cef..962b1893305 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -5,22 +5,34 @@ set -e # Bypass this test for valgrind test "$TESTUTIL_BYPASS_VALGRIND" = "1" && exit 0 -# Temporarily disabled # Smoke-test checkpoints as part of running "make check". -#echo "checkpoint: 3 mixed tables" -#$TEST_WRAPPER ./t -T 3 -t m -# Temporarily disabled -#echo "checkpoint: 6 column-store tables" -#$TEST_WRAPPER ./t -T 6 -t c +echo "checkpoint: 3 mixed tables" +$TEST_WRAPPER ./t -T 3 -t m -# Temporarily disabled -#echo "checkpoint: 6 LSM tables" -#$TEST_WRAPPER ./t -T 6 -t l +echo "checkpoint: 6 column-store tables" +$TEST_WRAPPER ./t -T 6 -t c -# Temporarily disabled -#echo "checkpoint: 6 mixed tables" -#$TEST_WRAPPER ./t -T 6 -t m +echo "checkpoint: 6 column-store tables, named checkpoint" +$TEST_WRAPPER ./t -c 'TeSt' -T 6 -t c + +echo "checkpoint: 6 column-store tables with prepare" +$TEST_WRAPPER ./t -T 6 -t c -p + +echo "checkpoint: 6 column-store tables, named checkpoint with prepare" +$TEST_WRAPPER ./t -c 'TeSt' -T 6 -t c -p + +echo "checkpoint: column-store tables, stress history store. Sweep and timestamps" +$TEST_WRAPPER ./t -t c -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB + +echo "checkpoint: column-store tables, Sweep and timestamps" +$TEST_WRAPPER ./t -t c -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB + +echo "checkpoint: 6 LSM tables" +$TEST_WRAPPER ./t -T 6 -t l + +echo "checkpoint: 6 mixed tables" +$TEST_WRAPPER ./t -T 6 -t m echo "checkpoint: 6 row-store tables" $TEST_WRAPPER ./t -T 6 -t r @@ -34,18 +46,14 @@ $TEST_WRAPPER ./t -T 6 -t r -p echo "checkpoint: 6 row-store tables, named checkpoint with prepare" $TEST_WRAPPER ./t -c 'TeSt' -T 6 -t r -p -# Temporarily disabled -#echo "checkpoint: row-store tables, stress history store. Sweep and timestamps" -#$TEST_WRAPPER ./t -t r -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB +echo "checkpoint: row-store tables, stress history store. Sweep and timestamps" +$TEST_WRAPPER ./t -t r -W 3 -r 2 -D -s -x -n 100000 -k 100000 -C cache_size=100MB -# Temporarily disabled -#echo "checkpoint: row-store tables, Sweep and timestamps" -#$TEST_WRAPPER ./t -t r -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB +echo "checkpoint: row-store tables, Sweep and timestamps" +$TEST_WRAPPER ./t -t r -W 3 -r 2 -s -x -n 100000 -k 100000 -C cache_size=100MB -# Temporarily disabled -#echo "checkpoint: 3 mixed tables, with sweep" -#$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -s -n 100000 -k 100000 +echo "checkpoint: 3 mixed tables, with sweep" +$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -s -n 100000 -k 100000 -# Temporarily disabled -#echo "checkpoint: 3 mixed tables, with timestamps" -#$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -x -n 100000 -k 100000 +echo "checkpoint: 3 mixed tables, with timestamps" +$TEST_WRAPPER ./t -T 3 -t m -W 3 -r 2 -x -n 100000 -k 100000 diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index b236cb14308..90378cc0de9 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -140,6 +140,8 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(g.home, 512, working_dir); + g.ts_stable = 0; + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { cleanup(cnt == 1); /* Clean up previous runs */ @@ -171,7 +173,7 @@ main(int argc, char *argv[]) free(g.cookies); g.cookies = NULL; if ((ret = wt_shutdown()) != 0) { - (void)log_print_err("Start workers failed", ret, 1); + (void)log_print_err("Shutdown failed", ret, 1); break; } } @@ -250,7 +252,6 @@ cleanup(bool remove_dir) g.running = 0; g.ntables_created = 0; g.ts_oldest = 0; - g.ts_stable = 0; if (remove_dir) testutil_make_work_dir(g.home); diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index 2cd947fbd79..05b9a83b75b 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -280,11 +280,12 @@ real_worker(void) } else testutil_check(__wt_snprintf( buf, sizeof(buf), "commit_timestamp=%x", g.ts_stable + 1)); - __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); if ((ret = session->commit_transaction(session, buf)) != 0) { + __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); (void)log_print_err("real_worker:commit_transaction", ret, 1); goto err; } + __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); start_txn = true; /* Occasionally reopen cursors after committing. */ if (next_rnd % 13 == 0) { diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index c863f47cbd3..26789a1ac9b 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -162,15 +162,9 @@ thread_run(void *arg) testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); -#if 0 - /* - * Make sure that alternative threads operate on column-store table - * - * FIXME-WT-6125: temporarily turn off column store test. - */ + /* Make alternate threads operate on the column-store table. */ if (td->id % 2 != 0) columnar_table = true; -#endif if (columnar_table) testutil_check(session->open_cursor(session, col_uri, NULL, NULL, &cursor)); @@ -384,7 +378,6 @@ recover_and_verify(uint32_t nthreads) fatal = false; for (i = 0; i < nthreads; ++i) { -#if 0 /* * Every alternative thread is operated on column-store table. Make sure that proper cursor * is used for verification of recovered records. @@ -396,11 +389,6 @@ recover_and_verify(uint32_t nthreads) columnar_table = false; cursor = row_cursor; } -#else - /* FIXME-WT-6125: temporarily turn off column store test. */ - columnar_table = false; - cursor = row_cursor; -#endif middle = 0; testutil_check(__wt_snprintf(fname[DELETE_RECORD_FILE_ID], diff --git a/src/third_party/wiredtiger/test/cursor_order/Makefile.am b/src/third_party/wiredtiger/test/cursor_order/Makefile.am index f5330b57d3b..448f8f95772 100644 --- a/src/third_party/wiredtiger/test/cursor_order/Makefile.am +++ b/src/third_party/wiredtiger/test/cursor_order/Makefile.am @@ -9,7 +9,7 @@ cursor_order_LDADD = $(top_builddir)/test/utility/libtest_util.la cursor_order_LDADD +=$(top_builddir)/libwiredtiger.la cursor_order_LDFLAGS = -static -TESTS = $(noinst_PROGRAMS) +TESTS = smoke.sh clean-local: rm -rf WT_TEST core.* *.core diff --git a/src/third_party/wiredtiger/test/cursor_order/smoke.sh b/src/third_party/wiredtiger/test/cursor_order/smoke.sh new file mode 100755 index 00000000000..13165c82d3d --- /dev/null +++ b/src/third_party/wiredtiger/test/cursor_order/smoke.sh @@ -0,0 +1,14 @@ +#! /bin/sh + +set -e + +# Smoke-test cursors as part of running "make check". + +echo "cursor_order: rows" +$TEST_WRAPPER ./cursor_order -tr + +echo "cursor_order: variable-length columns" +$TEST_WRAPPER ./cursor_order -tv + +echo "cursor_order: fixed-length columns" +$TEST_WRAPPER ./cursor_order -tf diff --git a/src/third_party/wiredtiger/test/suite/test_bug022.py b/src/third_party/wiredtiger/test/suite/test_bug022.py index 31fb2387662..8e7b355133e 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug022.py +++ b/src/third_party/wiredtiger/test/suite/test_bug022.py @@ -30,27 +30,37 @@ # Testing that we don't allow modifies on top of tombstone updates. import wiredtiger, wttest +from wtscenario import make_scenarios class test_bug022(wttest.WiredTigerTestCase): uri = 'file:test_bug022' conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def get_key(self, i): + return str(i) if self.usestrings else i + def test_apply_modifies_on_onpage_tombstone(self): - self.session.create(self.uri, 'key_format=S,value_format=S') + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) value = 'a' * 500 for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value + cursor[self.get_key(i)] = value self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Apply tombstones for every key. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) cursor.remove() self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) @@ -59,11 +69,11 @@ class test_bug022(wttest.WiredTigerTestCase): # Now try to apply a modify on top of the tombstone at timestamp 3. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 0, 100)]), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() # Check that the tombstone is visible. for i in range(1, 10000): - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index deea2f3dfc5..ea3fe771169 100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -28,6 +28,7 @@ from helper import copy_wiredtiger_home import wiredtiger, wttest +from wtscenario import make_scenarios # test_durable_ts03.py # Check that the checkpoint honors the durable timestamp of updates. @@ -35,11 +36,17 @@ class test_durable_ts03(wttest.WiredTigerTestCase): conn_config = 'cache_size=10MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_durable_ts03(self): # Create a table. uri = 'table:test_durable_ts03' nrows = 3000 - self.session.create(uri, 'key_format=i,value_format=u') + self.session.create(uri, 'key_format={},value_format=u'.format(self.key_format)) valueA = b"aaaaa" * 100 valueB = b"bbbbb" * 100 valueC = b"ccccc" * 100 @@ -51,7 +58,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): # Load the data into the table. session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueA session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) @@ -65,7 +72,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): # Update all the values within transaction. Commit the transaction with # a durable timestamp newer than the stable timestamp. cursor = session.open_cursor(uri, None) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueB session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(150)) @@ -105,7 +112,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEqual(value, valueA) self.assertEquals(cursor.reset(), 0) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[i] = valueC session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(220)) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index 4af2004c162..be6dafc5c38 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -45,6 +45,10 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): nentries = 10 table_cnt = 3 + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] types = [ ('all', dict(use_stable='false', run_wt=0)), ('all+wt', dict(use_stable='false', run_wt=1)), @@ -56,7 +60,7 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): ('stable+wt', dict(use_stable='true', run_wt=1)), ('stable+wt2', dict(use_stable='true', run_wt=2)), ] - scenarios = make_scenarios(types) + scenarios = make_scenarios(key_format_values, types) def data_and_checkpoint(self): # @@ -64,10 +68,11 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): # Add data to each of them separately and checkpoint so that each one # has a different stable timestamp. # - self.session.create(self.oplog_uri, 'key_format=i,value_format=i') - self.session.create(self.coll1_uri, 'key_format=i,value_format=i,log=(enabled=false)') - self.session.create(self.coll2_uri, 'key_format=i,value_format=i,log=(enabled=false)') - self.session.create(self.coll3_uri, 'key_format=i,value_format=i,log=(enabled=false)') + basecfg = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.oplog_uri, basecfg) + self.session.create(self.coll1_uri, basecfg + ',log=(enabled=false)') + self.session.create(self.coll2_uri, basecfg + ',log=(enabled=false)') + self.session.create(self.coll3_uri, basecfg + ',log=(enabled=false)') c_op = self.session.open_cursor(self.oplog_uri) c = [] c.append(self.session.open_cursor(self.coll1_uri)) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp11.py b/src/third_party/wiredtiger/test/suite/test_timestamp11.py index c4b4210df43..d5998bdc212 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp11.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp11.py @@ -32,14 +32,24 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + def test_timestamp_range(self): base = 'timestamp11' uri = 'file:' + base - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + + key = 'key' if self.usestrings else 1 + key2 = 'key2' if self.usestrings else 2 # Test that mixed timestamp usage where some transactions use timestamps # and others don't behave in the expected way. @@ -49,8 +59,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(2)) - c['key'] = 'value2' - c['key2'] = 'value2' + c[key] = 'value2' + c[key2] = 'value2' self.session.commit_transaction() c.close() @@ -62,13 +72,13 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(5)) - c['key'] = 'value5' + c[key] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key2'] = 'valueNOTS' + c[key2] = 'valueNOTS' self.session.commit_transaction() c.close() @@ -85,15 +95,15 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - self.assertEquals(c['key'], 'value2') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'value2') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + stable_ts) - self.assertEquals(c['key'], 'value2') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'value2') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() @@ -104,13 +114,13 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(5)) - c['key2'] = 'value5' + c[key2] = 'value5' self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c['key'] = 'valueNOTS' + c[key] = 'valueNOTS' self.session.commit_transaction() c.close() @@ -119,8 +129,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # Without a timestamp. We should see the latest value for each. c = self.session.open_cursor(uri) self.session.begin_transaction() - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'value5') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'value5') self.session.commit_transaction() c.close() @@ -128,8 +138,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # value at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + stable_ts) - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'valueNOTS') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'valueNOTS') self.session.commit_transaction() c.close() @@ -138,8 +148,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # we inserted at timestamp 5 after the non-timestamped insert. c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) - self.assertEquals(c['key'], 'valueNOTS') - self.assertEquals(c['key2'], 'value5') + self.assertEquals(c[key], 'valueNOTS') + self.assertEquals(c[key2], 'value5') self.session.commit_transaction() c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp12.py b/src/third_party/wiredtiger/test/suite/test_timestamp12.py index e54ed923474..5ccab50ce05 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp12.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp12.py @@ -38,12 +38,16 @@ class test_timestamp12(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' coll_uri = 'table:collection12' oplog_uri = 'table:oplog12' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] closecfg = [ ('dfl', dict(close_cfg='', all_expected=False)), ('use_stable', dict(close_cfg='use_timestamp=true', all_expected=False)), ('all_dirty', dict(close_cfg='use_timestamp=false', all_expected=True)), - ] - scenarios = make_scenarios(closecfg) + ] + scenarios = make_scenarios(key_format_values, closecfg) def verify_expected(self, op_exp, coll_exp): c_op = self.session.open_cursor(self.oplog_uri) @@ -67,8 +71,9 @@ class test_timestamp12(wttest.WiredTigerTestCase): # Add data to each of them separately and checkpoint so that each one # has a different stable timestamp. # - self.session.create(self.oplog_uri, 'key_format=i,value_format=i') - self.session.create(self.coll_uri, 'key_format=i,value_format=i,log=(enabled=false)') + basecfg = 'key_format={},value_format=i'.format(self.key_format) + self.session.create(self.oplog_uri, basecfg) + self.session.create(self.coll_uri, basecfg + ',log=(enabled=false)') c_op = self.session.open_cursor(self.oplog_uri) c_coll = self.session.open_cursor(self.coll_uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py index 8d6e753bf4d..3b4c4bf40ae 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp14.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -40,6 +40,12 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_all_durable_old(self): # This test was originally for testing the all_committed timestamp. # In the absence of prepared transactions, all_durable is identical to @@ -47,8 +53,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): all_durable_uri = self.uri + '_all_durable' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(all_durable_uri, 'key_format=i,value_format=i') - session2.create(all_durable_uri, 'key_format=i,value_format=i') + session1.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) # Scenario 0: No commit timestamp has ever been specified therefore # There is no all_durable timestamp and we will get an error @@ -132,8 +138,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): oldest_reader_uri = self.uri + '_oldest_reader_pinned' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(oldest_reader_uri, 'key_format=i,value_format=i') - session2.create(oldest_reader_uri, 'key_format=i,value_format=i') + session1.create(oldest_reader_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(oldest_reader_uri, 'key_format={},value_format=i'.format(self.key_format)) # Nothing is reading so there is no oldest reader. self.assertRaisesException(wiredtiger.WiredTigerError, @@ -190,7 +196,7 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): def test_pinned_oldest(self): pinned_oldest_uri = self.uri + 'pinned_oldest' session1 = self.setUpSessionOpen(self.conn) - session1.create(pinned_oldest_uri, 'key_format=i,value_format=i') + session1.create(pinned_oldest_uri, 'key_format={},value_format=i'.format(self.key_format)) # Confirm no oldest timestamp exists. self.assertRaisesException(wiredtiger.WiredTigerError, lambda: self.conn.query_timestamp('get=oldest')) @@ -241,7 +247,7 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): def test_all_durable(self): all_durable_uri = self.uri + '_all_durable' session1 = self.setUpSessionOpen(self.conn) - session1.create(all_durable_uri, 'key_format=i,value_format=i') + session1.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) # Since this is a non-prepared transaction, we'll be using the commit # timestamp when calculating all_durable since it's implied that they're @@ -329,8 +335,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): all_uri = self.uri + 'pinned_oldest' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(all_uri, 'key_format=i,value_format=i') - session2.create(all_uri, 'key_format=i,value_format=i') + session1.create(all_uri, 'key_format={},value_format=i'.format(self.key_format)) + session2.create(all_uri, 'key_format={},value_format=i'.format(self.key_format)) cur1 = session1.open_cursor(all_uri) cur2 = session2.open_cursor(all_uri) # Set up oldest timestamp. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp16.py b/src/third_party/wiredtiger/test/suite/test_timestamp16.py index f74557ad2c6..2485a14be12 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp16.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp16.py @@ -34,7 +34,6 @@ import random from suite_subprocess import suite_subprocess import wiredtiger, wttest -from wtscenario import make_scenarios class test_timestamp16(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_timestamp16' diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp17.py b/src/third_party/wiredtiger/test/suite/test_timestamp17.py index 3c2453b0aa2..fe4ec606d23 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp17.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp17.py @@ -43,8 +43,14 @@ class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def test_inconsistent_timestamping(self): - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) self.session.begin_transaction() cur1 = self.session.open_cursor(self.uri) cur1[1] = 1 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp18.py b/src/third_party/wiredtiger/test/suite/test_timestamp18.py index c7f20dbd9a4..b41fc5ccb34 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp18.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp18.py @@ -41,15 +41,23 @@ from wtscenario import make_scenarios class test_timestamp18(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] non_ts_writes = [ ('insert', dict(delete=False)), ('delete', dict(delete=True)), ] - scenarios = make_scenarios(non_ts_writes) + scenarios = make_scenarios(key_format_values, non_ts_writes) + + def get_key(self, i): + return str(i) if self.usestrings else i def test_ts_writes_with_non_ts_write(self): uri = 'table:test_timestamp18' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -61,17 +69,17 @@ class test_timestamp18(wttest.WiredTigerTestCase): # A series of timestamped writes on each key. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) # Add a non-timestamped delete. @@ -80,10 +88,10 @@ class test_timestamp18(wttest.WiredTigerTestCase): for i in range(1, 10000): if i % 2 == 0: if self.delete: - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) cursor.remove() else: - cursor[str(i)] = value4 + cursor[self.get_key(i)] = value4 self.session.checkpoint() @@ -94,16 +102,16 @@ class test_timestamp18(wttest.WiredTigerTestCase): # invisible. if i % 2 == 0: if self.delete: - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: - self.assertEqual(cursor[str(i)], value4) + self.assertEqual(cursor[self.get_key(i)], value4) # Otherwise, expect one of the timestamped writes. else: if ts == 2: - self.assertEqual(cursor[str(i)], value1) + self.assertEqual(cursor[self.get_key(i)], value1) elif ts == 3: - self.assertEqual(cursor[str(i)], value2) + self.assertEqual(cursor[self.get_key(i)], value2) else: - self.assertEqual(cursor[str(i)], value3) + self.assertEqual(cursor[self.get_key(i)], value3) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py index 9f5b71df843..e271c8f1145 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp19.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -30,15 +30,22 @@ # Use the oldest timestamp in the metadata as the oldest timestamp on restart. import wiredtiger, wttest from wtdataset import SimpleDataSet +from wtscenario import make_scenarios class test_timestamp19(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + def updates(self, uri, value, ds, nrows, commit_ts): session = self.session cursor = session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) @@ -46,11 +53,11 @@ class test_timestamp19(wttest.WiredTigerTestCase): def test_timestamp(self): uri = "table:test_timestamp19" - create_params = 'value_format=S,key_format=i' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_params) ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() nrows = 1000 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp20.py b/src/third_party/wiredtiger/test/suite/test_timestamp20.py index c4edcbfe592..7bfeb670004 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp20.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp20.py @@ -27,6 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_timestamp20.py # Exercise fixing up of out-of-order updates in the history store. @@ -34,9 +35,18 @@ class test_timestamp20(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('string-row', dict(key_format='S', usestrings=True)), + ('column', dict(key_format='r', usestrings=False)), + ] + scenarios = make_scenarios(key_format_values) + + def get_key(self, i): + return str(i) if self.usestrings else i + def test_timestamp20_standard(self): uri = 'table:test_timestamp20' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -48,17 +58,17 @@ class test_timestamp20(wttest.WiredTigerTestCase): for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) old_reader_session = self.conn.open_session() @@ -69,19 +79,19 @@ class test_timestamp20(wttest.WiredTigerTestCase): # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value4 + cursor[self.get_key(i)] = value4 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() - cursor[str(i)] = value5 + cursor[self.get_key(i)] = value5 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): - self.assertEqual(cursor[str(i)], value4) + self.assertEqual(cursor[self.get_key(i)], value4) self.session.rollback_transaction() for i in range(1, 10000): - self.assertEqual(old_reader_cursor[str(i)], value2) + self.assertEqual(old_reader_cursor[self.get_key(i)], value2) old_reader_session.rollback_transaction() # In this test we're using modifies since they are more sensitive to corruptions. @@ -90,7 +100,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # the conversion to a Python string. def test_timestamp20_modify(self): uri = 'table:test_timestamp20' - self.session.create(uri, 'key_format=S,value_format=S') + self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -101,19 +111,19 @@ class test_timestamp20(wttest.WiredTigerTestCase): # Apply the base value. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value1 + cursor[self.get_key(i)] = value1 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Now apply a series of modifies. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) @@ -129,7 +139,7 @@ class test_timestamp20(wttest.WiredTigerTestCase): # This will be the end of the chain of modifies. for i in range(1, 10000): self.session.begin_transaction() - cursor.set_key(str(i)) + cursor.set_key(self.get_key(i)) self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(40)) @@ -137,17 +147,17 @@ class test_timestamp20(wttest.WiredTigerTestCase): # correction to the existing contents. for i in range(1, 10000): self.session.begin_transaction() - cursor[str(i)] = value2 + cursor[self.get_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(25)) self.session.begin_transaction() - cursor[str(i)] = value3 + cursor[self.get_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50)) # Open up a new transaction and read at 30. # We shouldn't be able to see past 5 due to txnid visibility. self.session.begin_transaction('read_timestamp=' + self.timestamp_str(30)) for i in range(1, 10000): - self.assertEqual(cursor[str(i)], value2) + self.assertEqual(cursor[self.get_key(i)], value2) self.session.rollback_transaction() # Put together expected value. @@ -157,5 +167,5 @@ class test_timestamp20(wttest.WiredTigerTestCase): # On the other hand, this older transaction SHOULD be able to read past the 5. for i in range(1, 10000): - self.assertEqual(old_reader_cursor[str(i)], expected) + self.assertEqual(old_reader_cursor[self.get_key(i)], expected) old_reader_session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index e02dbf99e09..b26b844dd32 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -31,6 +31,7 @@ import wiredtiger, wttest, re, suite_random from wtdataset import SimpleDataSet from contextlib import contextmanager +from wtscenario import make_scenarios class test_timestamp22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' @@ -46,6 +47,12 @@ class test_timestamp22(wttest.WiredTigerTestCase): SUCCESS = 'success' FAILURE = 'failure' + key_format_values = [ + ('integer-row', dict(key_format='i')), + ('column', dict(key_format='r')), + ] + scenarios = make_scenarios(key_format_values) + # Control execution of an operation, looking for exceptions and error messages. # Usage: # with self.expect(self.FAILURE, 'some operation'): @@ -240,7 +247,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): msg = 'inserts with commit config(' + commit_config + ')' try: - for i in range(0, self.nrows): + for i in range(1, self.nrows + 1): needs_rollback = False if self.do_illegal(): # Illegal outside of transaction @@ -380,14 +387,14 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: iterations = 1000 - create_params = 'value_format=S,key_format=i' + create_params = 'key_format={},value_format=S'.format(self.key_format) self.session.create(self.uri, create_params) self.set_global_timestamps(1, 1, -1, -1) # Create tables with no entries ds = SimpleDataSet( - self, self.uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, self.uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') # We do a bunch of iterations, doing transactions, prepare, and global timestamp calls # with timestamps that are sometimes valid, sometimes not. We use the iteration number @@ -430,7 +437,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Make sure the resulting rows are what we expect. cursor = self.session.open_cursor(self.uri) - expect_key = 0 + expect_key = 1 expect_value = self.commit_value for k,v in cursor: self.assertEquals(k, expect_key) @@ -440,7 +447,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Although it's theoretically possible to never successfully update a single row, # with a large number of iterations that should never happen. I'd rather catch # a test code error where we mistakenly don't update any rows. - self.assertGreater(expect_key, 0) + self.assertGreater(expect_key, 1) cursor.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp23.py b/src/third_party/wiredtiger/test/suite/test_timestamp23.py new file mode 100644 index 00000000000..60d97cf78fb --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp23.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_timestamp23.py +# +# delete keys repeatedly at successive timestamps +class test_timestamp23(wttest.WiredTigerTestCase): + conn_config = '' + session_config = 'isolation=snapshot' + + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + + def test_timestamp(self): + + # Create a file that contains active history (content newer than the oldest timestamp). + table_uri = 'table:timestamp23' + ds = SimpleDataSet( + self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') + ds.populate() + self.session.checkpoint() + + key = 5 + value_1 = 'a' * 500 + value_2 = 'b' * 500 + value_3 = 'c' * 500 + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + cursor = self.session.open_cursor(ds.uri) + + # Write two values at timestamp 10. We'll muck with the first value + # and use the second to reference the page for eviction. + self.session.begin_transaction('read_timestamp=10') + cursor[key] = value_1 + cursor[key+1] = value_2 + self.session.commit_transaction('commit_timestamp=11') + + # Delete the first value at timestamp 20. + self.session.begin_transaction('read_timestamp=20') + cursor.set_key(key) + cursor.remove() + self.session.commit_transaction('commit_timestamp=21') + + # Put it back at timestamp 30. + self.session.begin_transaction('read_timestamp=30') + cursor[key] = value_3 + self.session.commit_transaction('commit_timestamp=31') + + # Delete it again at timestamp 40. + self.session.begin_transaction('read_timestamp=40') + cursor.set_key(key) + cursor.remove() + self.session.commit_transaction('commit_timestamp=41') + + # Evict the page using the second key. + evict_cursor = self.session.open_cursor(ds.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + v = evict_cursor[key+1] + self.assertEqual(v, value_2) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + + # Create a separate session and a cursor to read the original value at timestamp 12. + session2 = self.conn.open_session() + cursor2 = session2.open_cursor(ds.uri) + session2.begin_transaction('read_timestamp=12') + v = cursor2[key] + self.assertEqual(v, value_1) + + self.session.breakpoint() + + # Now delete the original value. This _should_ cause WT_ROLLBACK, but with a column + # store bug seen and fixed in August 2021, it succeeds, and the resulting invalid + # tombstone will cause reconciliation to assert. (To see this behavior, comment out the + # self.fail call and let the transaction commit.) + try: + cursor2.remove() + self.fail("Conflicting remove did not fail") + session2.commit_transaction('commit_timestamp=50') + except wiredtiger.WiredTigerError as e: + self.assertTrue(wiredtiger.wiredtiger_strerror(wiredtiger.WT_ROLLBACK) in str(e)) + + cursor.close() + cursor2.close() + +if __name__ == '__main__': + wttest.run() |