diff options
26 files changed, 555 insertions, 140 deletions
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 1b4cf50716e..1b0afc64eda 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -457,6 +457,7 @@ WTPERF WaitForSingleObject WakeAllConditionVariable Wconditional +Wduplicated WeakHashLen Werror Wformat diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 6ca94472236..9b01b52a1e1 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "b85c4b8a97b80cd0eb5e71843fdd12c0b905c5a7" + "commit": "823abc9cb3d952b59276381a6181f5b1a0749cc9" } diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 9a157903053..8888fa61ae7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -434,7 +434,7 @@ __wt_sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) uint64_t internal_bytes, internal_pages, leaf_bytes, leaf_pages; uint64_t oldest_id, saved_pinned_id, time_start, time_stop; uint32_t flags, rec_flags; - bool dirty, is_hs, timer, tried_eviction; + bool dirty, internal_cleanup, is_hs, timer, tried_eviction; conn = S2C(session); btree = S2BT(session); @@ -557,6 +557,20 @@ __wt_sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) /* Read pages with history store entries and evict them asap. */ LF_SET(WT_READ_WONT_NEED); + /* + * Perform checkpoint cleanup when not in startup or shutdown phase by traversing internal + * pages looking for obsolete child pages. This is row-store specific, column-store pages + * cannot be discarded and must be rewritten as they contain chunks of the name space. For + * the same reason, only read in-memory pages when doing column-store checkpoints (row-store + * reads all of the internal pages to improve cleanup). + */ + if (btree->type == BTREE_ROW) + internal_cleanup = !F_ISSET(conn, WT_CONN_RECOVERING | WT_CONN_CLOSING_TIMESTAMP); + else { + LF_SET(WT_READ_CACHE); + internal_cleanup = false; + } + for (;;) { WT_ERR(__sync_dup_walk(session, walk, flags, &prev)); WT_ERR(__wt_tree_walk_custom_skip(session, &walk, __sync_page_skip, NULL, flags)); @@ -564,12 +578,7 @@ __wt_sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) if (walk == NULL) break; - /* - * Perform checkpoint cleanup when not in startup or shutdown phase by traversing - * through internal pages looking for obsolete child pages. - */ - if (!F_ISSET(conn, WT_CONN_RECOVERING | WT_CONN_CLOSING_TIMESTAMP) && - F_ISSET(walk, WT_REF_FLAG_INTERNAL)) { + if (F_ISSET(walk, WT_REF_FLAG_INTERNAL) && internal_cleanup) { WT_WITH_PAGE_INDEX( session, ret = __sync_ref_int_obsolete_cleanup(session, walk, &ref_list)); WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index e3ab3fecc16..8c0c2bcb305 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -1190,13 +1190,13 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * updates first, the history search logic may race with other sessions modifying the same key * and checkpoint moving the new updates to the history store. * - * For prepared delete commit, we don't need to fix the history store. Whereas for rollback, if - * the update is also from the same prepared transaction, restore the update from history store - * or remove the key. + * Fix the history store entry for the updates other than tombstone type or the tombstone + * followed by the update is also from the same prepared transaction by either restoring the + * previous update from history store or removing the key. */ prepare_on_disk = F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS) && (upd->type != WT_UPDATE_TOMBSTONE || - (!commit && upd->next != NULL && upd->durable_ts == upd->next->durable_ts && + (upd->next != NULL && upd->durable_ts == upd->next->durable_ts && upd->txnid == upd->next->txnid && upd->start_ts == upd->next->start_ts)); first_committed_upd_in_hs = first_committed_upd != NULL && F_ISSET(first_committed_upd, WT_UPDATE_HS); diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 6004ddd3db2..fb85adc9745 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -449,10 +449,16 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page * If we have fixed the out-of-order timestamps, then the newer update reinserted with an * older timestamp may have a durable timestamp that is smaller than the current stop * durable timestamp. + * + * It is possible that there can be an update in the history store with a max stop timestamp + * in the middle of the same key updates. This occurs when the checkpoint writes the + * committed prepared update and further updates on that key including the history store + * changes before the transaction fixes the history store update to have a proper stop + * timestamp. It is a rare scenario. */ WT_ASSERT(session, hs_stop_durable_ts <= newer_hs_durable_ts || hs_start_ts == hs_stop_durable_ts || - hs_start_ts == newer_hs_durable_ts || first_record); + hs_start_ts == newer_hs_durable_ts || first_record || hs_stop_durable_ts == WT_TS_MAX); if (hs_stop_durable_ts < newer_hs_durable_ts) WT_STAT_CONN_DATA_INCR(session, txn_rts_hs_stop_older_than_newer_start); @@ -816,11 +822,10 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r /* * If we found a stable update on the insert list, this key needs no further attention. * Any other keys in this cell with stable updates also do not require attention. But - * beyond that, the on-disk value must be older than - * the update we found. That means it too is stable(*), so any keys in the cell that - * _don't_ have stable updates on the update list don't need further attention either. - * (And any unstable updates were just handled above.) Thus we can skip iterating over - * the cell. + * beyond that, the on-disk value must be older than the update we found. That means it + * too is stable(*), so any keys in the cell that _don't_ have stable updates on the + * update list don't need further attention either. (And any unstable updates were just + * handled above.) Thus we can skip iterating over the cell. * * Furthermore, if the cell is deleted it must be * itself stable, because cells only appear as deleted if there is no older value that diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index f5da81e75ed..f0935ba2c05 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -57,11 +57,11 @@ all_TESTS += test_wt2323_join_visibility test_wt2535_insert_race_SOURCES = wt2535_insert_race/main.c noinst_PROGRAMS += test_wt2535_insert_race -all_TESTS += test_wt2535_insert_race +all_TESTS += wt2535_insert_race/smoke.sh test_wt2447_join_main_table_SOURCES = wt2447_join_main_table/main.c noinst_PROGRAMS += test_wt2447_join_main_table -all_TESTS += test_wt2447_join_main_table +all_TESTS += wt2447_join_main_table/smoke.sh test_wt2695_checksum_SOURCES = wt2695_checksum/main.c noinst_PROGRAMS += test_wt2695_checksum @@ -81,11 +81,11 @@ all_TESTS += test_wt2834_join_bloom_fix test_wt2853_perf_SOURCES = wt2853_perf/main.c noinst_PROGRAMS += test_wt2853_perf -all_TESTS += test_wt2853_perf +all_TESTS += wt2853_perf/smoke.sh test_wt2909_checkpoint_integrity_SOURCES = wt2909_checkpoint_integrity/main.c noinst_PROGRAMS += test_wt2909_checkpoint_integrity -all_TESTS += test_wt2909_checkpoint_integrity +all_TESTS += wt2909_checkpoint_integrity/smoke.sh test_wt2999_join_extractor_SOURCES = wt2999_join_extractor/main.c noinst_PROGRAMS += test_wt2999_join_extractor @@ -117,7 +117,7 @@ all_TESTS += test_wt3874_pad_byte_collator test_wt4105_large_doc_small_upd_SOURCES = wt4105_large_doc_small_upd/main.c noinst_PROGRAMS += test_wt4105_large_doc_small_upd -all_TESTS += test_wt4105_large_doc_small_upd +all_TESTS += wt4105_large_doc_small_upd/smoke.sh test_wt4117_checksum_SOURCES = wt4117_checksum/main.c noinst_PROGRAMS += test_wt4117_checksum diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index acf14348c0a..072192eee0a 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -75,7 +75,8 @@ static const char *const uri_collection = "table:collection"; static const char *const ckpt_file = "checkpoint_done"; -static bool compat, inmem, stable_set, use_columns, use_ts, use_txn; +static bool compat, inmem, use_columns, use_ts, use_txn; +static volatile bool stable_set; static volatile uint64_t global_ts = 1; static volatile uint64_t uid = 1; typedef struct { diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 1d59222104b..c9883800199 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -149,10 +149,14 @@ thread_ts_run(void *arg) WT_RAND_STATE rnd; WT_SESSION *session; THREAD_DATA *td; + wt_timestamp_t all_dur_ts, prev_all_dur_ts; + uint32_t rand_op; int dbg; char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE]; bool first; + prev_all_dur_ts = WT_TS_NONE; + td = (THREAD_DATA *)arg; __wt_random_init(&rnd); @@ -168,16 +172,39 @@ thread_ts_run(void *arg) ret = td->conn->query_timestamp(td->conn, ts_string, "get=all_durable"); testutil_check(pthread_rwlock_unlock(&ts_lock)); testutil_assert(ret == 0 || ret == WT_NOTFOUND); + /* + * All durable can intermittently move backwards, we do not want to set stable and the + * oldest timestamps backwards - refer WT-8001. + */ + all_dur_ts = strtoul(ts_string, NULL, 16); + if (!first && all_dur_ts < prev_all_dur_ts) { + __wt_sleep(0, 1000); + continue; + } if (ret == 0) { - /* Periodically let the oldest timestamp lag. */ - if (!first && __wt_random(&rnd) % 4 == 0) + rand_op = __wt_random(&rnd) % 4; + /* + * Periodically let the oldest timestamp lag. Other times set the stable and oldest + * timestamps as separate API calls. The rest of the time set them both as one call. + */ + if (rand_op == 0) { testutil_check( __wt_snprintf(tscfg, sizeof(tscfg), "stable_timestamp=%s", ts_string)); - else - testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), - "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string)); - testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "oldest_timestamp=%s", ts_string)); + testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + } else { + if (!first && rand_op == 1) + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "stable_timestamp=%s", ts_string)); + else + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string)); + testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + } first = false; + prev_all_dur_ts = all_dur_ts; /* * Set and reset the checkpoint retention setting on a regular basis. We want to test * racing with the internal archive thread while we're here. diff --git a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c index 480d49e1344..67ce0c081c0 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c @@ -86,16 +86,29 @@ main(int argc, char *argv[]) WT_CURSOR *cursor1, *cursor2, *jcursor; WT_ITEM d; WT_SESSION *session; - uint64_t maincount; + uint64_t maincount, i64; int half, i, j; - char bloom_cfg[128], index1uri[256], index2uri[256], joinuri[256]; + char bloom_cfg[128], index1uri[256], index2uri[256], joinuri[256], table_cfg[128]; const char *tablename; opts = &_opts; memset(opts, 0, sizeof(*opts)); + /* 0 isn't a valid table_type; use rows by default */ + opts->table_type = TABLE_ROW; testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); + switch (opts->table_type) { + case TABLE_COL: + printf("Table type: columns\n"); + break; + case TABLE_FIX: + testutil_die(ENOTSUP, "Fixed-length column store not supported"); + case TABLE_ROW: + printf("Table type: rows\n"); + break; + } + tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; @@ -106,8 +119,10 @@ main(int argc, char *argv[]) testutil_check(wiredtiger_open(opts->home, NULL, "statistics=(all),create", &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check( - session->create(session, opts->uri, "key_format=i,value_format=iiu,columns=(k,v1,v2,d)")); + testutil_check(__wt_snprintf(table_cfg, sizeof(table_cfg), + "key_format=%s,value_format=iiu,columns=(k,v1,v2,d)", + opts->table_type == TABLE_ROW ? "i" : "r")); + testutil_check(session->create(session, opts->uri, table_cfg)); testutil_check(session->create(session, index1uri, "columns=(v1)")); testutil_check(session->create(session, index2uri, "columns=(v2)")); @@ -117,7 +132,7 @@ main(int argc, char *argv[]) d.data = dmalloc(d.size); memset((char *)d.data, 7, d.size); - for (i = 0; i < N_RECORDS; ++i) { + for (i = 1; i < N_RECORDS + 1; ++i) { cursor1->set_key(cursor1, i); cursor1->set_value(cursor1, i, i, &d); testutil_check(cursor1->insert(cursor1)); @@ -150,13 +165,18 @@ main(int argc, char *argv[]) /* Expect one value returned */ testutil_assert(jcursor->next(jcursor) == 0); i = 0; - testutil_assert(jcursor->get_key(jcursor, &i) == 0); - testutil_assert(i == (int)half); + if (opts->table_type == TABLE_ROW) + testutil_assert(jcursor->get_key(jcursor, &i) == 0); + else { + testutil_assert(jcursor->get_key(jcursor, &i64) == 0); + i = (int)i64; + } + testutil_assert(i == half); i = j = 0; memset(&d, 0, sizeof(d)); testutil_assert(jcursor->get_value(jcursor, &i, &j, &d) == 0); - testutil_assert(i == (int)half); - testutil_assert(j == (int)half); + testutil_assert(i == half); + testutil_assert(j == half); testutil_assert(d.size == 4100); for (i = 0; i < 4100; i++) testutil_assert(((char *)d.data)[i] == 7); diff --git a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/smoke.sh b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/smoke.sh new file mode 100755 index 00000000000..c57694807ac --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/smoke.sh @@ -0,0 +1,21 @@ +#! /bin/sh + +set -e + +# Smoke-test wt2447_join_main_table as part of running "make check". + +if [ -n "$1" ] +then + # If the test binary is passed in manually. + test_bin=$1 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_wt2447_join_main_table +fi + +$TEST_WRAPPER $test_bin -t r +$TEST_WRAPPER $test_bin -t c + diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c index 8f8fc3a9015..4bc8b7d985e 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c @@ -38,6 +38,48 @@ void *thread_insert_race(void *); static uint64_t ready_counter; +/* + * set_key -- + * Wrapper providing the correct typing for the WT_CURSOR::set_key variadic argument. + */ +static void +set_key(WT_CURSOR *c, uint64_t value) +{ + c->set_key(c, value); +} + +/* + * set_value -- + * Wrapper providing the correct typing for the WT_CURSOR::set_value variadic argument. + */ +static void +set_value(TEST_OPTS *opts, WT_CURSOR *c, uint64_t value) +{ + if (opts->table_type == TABLE_FIX) + c->set_value(c, (uint8_t)value); + else + c->set_value(c, value); +} + +/* + * get_value -- + * Wrapper providing the correct typing for the WT_CURSOR::get_value variadic argument. + */ +static uint64_t +get_value(TEST_OPTS *opts, WT_CURSOR *c) +{ + uint64_t value64; + uint8_t value8; + + if (opts->table_type == TABLE_FIX) { + testutil_check(c->get_value(c, &value8)); + return (value8); + } else { + testutil_check(c->get_value(c, &value64)); + return (value64); + } +} + int main(int argc, char *argv[]) { @@ -46,8 +88,9 @@ main(int argc, char *argv[]) WT_SESSION *session; clock_t ce, cs; pthread_t id[100]; - uint64_t current_value; + uint64_t current_value, expected_value; int i; + char tableconf[128]; /* Bypass this test for valgrind */ if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) @@ -64,13 +107,15 @@ main(int argc, char *argv[]) testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=2G,eviction=(threads_max=5),statistics=(fast)", &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check( - session->create(session, opts->uri, "key_format=Q,value_format=Q,leaf_page_max=32k,")); + testutil_check(__wt_snprintf(tableconf, sizeof(tableconf), + "key_format=%s,value_format=%s,leaf_page_max=32k,", opts->table_type == TABLE_ROW ? "Q" : "r", + opts->table_type == TABLE_FIX ? "8t" : "Q")); + testutil_check(session->create(session, opts->uri, tableconf)); /* Create the single record. */ testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &c)); - c->set_key(c, 1); - c->set_value(c, 0); + set_key(c, 1); + set_value(opts, c, 0); testutil_check(c->insert(c)); testutil_check(c->close(c)); cs = clock(); @@ -80,10 +125,13 @@ main(int argc, char *argv[]) while (--i >= 0) testutil_check(pthread_join(id[i], NULL)); testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &c)); - c->set_key(c, 1); + set_key(c, 1); testutil_check(c->search(c)); - testutil_check(c->get_value(c, ¤t_value)); - if (current_value != opts->nthreads * opts->nrecords) { + current_value = get_value(opts, c); + expected_value = opts->nthreads * opts->nrecords; + if (opts->table_type == TABLE_FIX) + expected_value %= 256; + if (current_value != expected_value) { fprintf(stderr, "ERROR: didn't get expected number of changes\n"); fprintf(stderr, "got: %" PRIu64 ", expected: %" PRIu64 "\n", current_value, opts->nthreads * opts->nrecords); @@ -129,11 +177,11 @@ thread_insert_race(void *arg) for (i = 0; i < opts->nrecords; ++i) { testutil_check(session->begin_transaction(session, "isolation=snapshot")); - cursor->set_key(cursor, 1); + set_key(cursor, 1); testutil_check(cursor->search(cursor)); - testutil_check(cursor->get_value(cursor, &value)); - cursor->set_key(cursor, 1); - cursor->set_value(cursor, value + 1); + value = get_value(opts, cursor); + set_key(cursor, 1); + set_value(opts, cursor, value + 1); if ((ret = cursor->update(cursor)) != 0) { if (ret == WT_ROLLBACK) { testutil_check(session->rollback_transaction(session, NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/smoke.sh b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/smoke.sh new file mode 100755 index 00000000000..9db537df92b --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/smoke.sh @@ -0,0 +1,21 @@ +#! /bin/sh + +set -e + +# Smoke-test wt2535_insert_race as part of running "make check". + +if [ -n "$1" ] +then + # If the test binary is passed in manually. + test_bin=$1 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_wt2535_insert_race +fi + +$TEST_WRAPPER $test_bin -t r +$TEST_WRAPPER $test_bin -t c +$TEST_WRAPPER $test_bin -t f diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index c9a6924e217..9dff68e1076 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -62,6 +62,7 @@ typedef struct { char baluri[256]; char flaguri[256]; bool bloom; + bool usecolumns; } SHARED_OPTS; typedef struct { @@ -83,7 +84,8 @@ main(int argc, char *argv[]) WT_CURSOR *maincur; WT_SESSION *session; pthread_t get_tid[N_GET_THREAD], insert_tid[N_INSERT_THREAD]; - int i, nfail; + int i, key, nfail; + char tableconf[128]; const char *tablename; /* @@ -96,6 +98,7 @@ main(int argc, char *argv[]) opts = &_opts; sharedopts = &_sharedopts; memset(opts, 0, sizeof(*opts)); + opts->table_type = TABLE_ROW; memset(sharedopts, 0, sizeof(*sharedopts)); memset(insert_args, 0, sizeof(insert_args)); memset(get_args, 0, sizeof(get_args)); @@ -103,6 +106,10 @@ main(int argc, char *argv[]) sharedopts->bloom = BLOOM; testutil_check(testutil_parse_opts(argc, argv, opts)); + if (opts->table_type == TABLE_FIX) + testutil_die(ENOTSUP, "Fixed-length column store not supported"); + sharedopts->usecolumns = (opts->table_type == TABLE_COL); + testutil_make_work_dir(opts->home); testutil_progress(opts, "start"); @@ -116,8 +123,10 @@ main(int argc, char *argv[]) * Note: id is repeated as id2. This makes it easier to identify the primary key in dumps of the * index files. */ - testutil_check(session->create( - session, opts->uri, "key_format=i,value_format=iiSii,columns=(id,post,bal,extra,flag,id2)")); + testutil_check(__wt_snprintf(tableconf, sizeof(tableconf), + "key_format=%s,value_format=iiSii,columns=(id,post,bal,extra,flag,id2)", + sharedopts->usecolumns ? "r" : "i")); + testutil_check(session->create(session, opts->uri, tableconf)); tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); @@ -138,8 +147,17 @@ main(int argc, char *argv[]) * easier. */ testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); - maincur->set_key(maincur, N_RECORDS); - maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS); + /* + * Do not constant-fold this assignment: in gcc 10.3, if you pass the constant directly to + * set_key, -Wduplicated-branches fails to notice the type difference between the two cases and + * gives a spurious warning, and diagnostic builds fail. + */ + key = N_RECORDS + 1; + if (sharedopts->usecolumns) + maincur->set_key(maincur, (uint64_t)key); + else + maincur->set_key(maincur, key); + maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS + 1); testutil_check(maincur->insert(maincur)); testutil_check(maincur->close(maincur)); testutil_check(session->close(session, NULL)); @@ -199,6 +217,7 @@ main(int argc, char *argv[]) static void * thread_insert(void *arg) { + SHARED_OPTS *sharedopts; TEST_OPTS *opts; THREAD_ARGS *threadargs; WT_CURSOR *maincur; @@ -210,6 +229,7 @@ thread_insert(void *arg) threadargs = (THREAD_ARGS *)arg; opts = threadargs->testopts; + sharedopts = threadargs->sharedopts; testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); @@ -223,9 +243,12 @@ thread_insert(void *arg) /* * Insert threads may stomp on each other's records; that's okay. */ - key = (int)(__wt_random(&rnd) % N_RECORDS); + key = (int)(__wt_random(&rnd) % N_RECORDS) + 1; testutil_check(session->begin_transaction(session, NULL)); - maincur->set_key(maincur, key); + if (sharedopts->usecolumns) + maincur->set_key(maincur, (uint64_t)key); + else + maincur->set_key(maincur, key); if (__wt_random(&rnd) % 2 == 0) post = 54321; else @@ -299,7 +322,11 @@ thread_get(void *arg) testutil_check(postcur->get_value(postcur, &post2, &bal, &extra, &flag, &key)); testutil_assert((flag > 0 && bal < 0) || (flag == 0 && bal >= 0)); - maincur->set_key(maincur, key); + if (sharedopts->usecolumns) + maincur->set_key(maincur, (uint64_t)key); + else + maincur->set_key(maincur, key); + fflush(stdout); testutil_check(maincur->search(maincur)); testutil_check(maincur->get_value(maincur, &post2, &bal2, &extra, &flag2, &key2)); testutil_check(maincur->reset(maincur)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/smoke.sh b/src/third_party/wiredtiger/test/csuite/wt2853_perf/smoke.sh new file mode 100755 index 00000000000..d2351082329 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/smoke.sh @@ -0,0 +1,20 @@ +#! /bin/sh + +set -e + +# Smoke-test wt2853_perf as part of running "make check". + +if [ -n "$1" ] +then + # If the test binary is passed in manually. + test_bin=$1 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_wt2853_perf +fi + +$TEST_WRAPPER $test_bin -t r +$TEST_WRAPPER $test_bin -t c diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index e6111dc937b..ce64967197a 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -55,6 +55,8 @@ * The data is stored in two tables, one having indices. Both tables have the same keys and are * updated with the same key in a single transaction. * + * The keys are int (key_format 'i'); for column-store these are converted on the fly to uint64_t. + * * Failure mode: If one table is out of step with the other, that is detected as a failure at the * top level. If an index is missing values (or has extra values), that is likewise a failure at the * top level. If the tables or the home directory cannot be opened, that is a top level error. The @@ -102,7 +104,7 @@ check_results(TEST_OPTS *opts, uint64_t *foundp) { WT_CURSOR *maincur, *maincur2, *v0cur, *v1cur, *v2cur; WT_SESSION *session; - uint64_t count, idxcount, nrecords; + uint64_t count, idxcount, nrecords, key64; uint32_t rndint; int key, key_got, ret, v0, v1, v2; char *big, *bigref; @@ -121,7 +123,12 @@ check_results(TEST_OPTS *opts, uint64_t *foundp) count = 0; while ((ret = maincur->next(maincur)) == 0) { testutil_check(maincur2->next(maincur2)); - testutil_check(maincur2->get_key(maincur2, &key_got)); + if (opts->table_type == TABLE_ROW) + testutil_check(maincur2->get_key(maincur2, &key_got)); + else { + testutil_check(maincur2->get_key(maincur2, &key64)); + key_got = (int)key64; + } testutil_check(maincur2->get_value(maincur2, &rndint)); generate_key(count, &key); @@ -129,7 +136,12 @@ check_results(TEST_OPTS *opts, uint64_t *foundp) testutil_assert(key == key_got); /* Check the key/values in main table. */ - testutil_check(maincur->get_key(maincur, &key_got)); + if (opts->table_type == TABLE_ROW) + testutil_check(maincur->get_key(maincur, &key_got)); + else { + testutil_check(maincur->get_key(maincur, &key64)); + key_got = (int)key64; + } testutil_assert(key == key_got); check_values(maincur, v0, v1, v2, big); @@ -260,7 +272,7 @@ enable_failures(uint64_t allow_writes, uint64_t allow_reads) static void generate_key(uint64_t i, int *keyp) { - *keyp = (int)i; + *keyp = (int)i + 1; } /* @@ -313,6 +325,8 @@ run_check_subtest( subtest_args[narg++] = (char *)"-n"; testutil_check(__wt_snprintf(rarg, sizeof(rarg), "%" PRIu64, opts->nrecords)); subtest_args[narg++] = rarg; /* number of records */ + subtest_args[narg++] = (char *)"-t"; + subtest_args[narg++] = (char *)(opts->table_type == TABLE_ROW ? "r" : "c"); subtest_args[narg++] = NULL; testutil_assert(narg <= MAX_ARGS); if (opts->verbose) @@ -485,10 +499,11 @@ subtest_main(int argc, char *argv[], bool close_test) struct rlimit rlim; TEST_OPTS *opts, _opts; WT_SESSION *session; - char config[1024], filename[1024], buf[1024]; + char buf[1024], config[1024], filename[1024], tableconf[128]; opts = &_opts; memset(opts, 0, sizeof(*opts)); + opts->table_type = TABLE_ROW; memset(&rlim, 0, sizeof(rlim)); /* No core files during fault injection tests. */ @@ -513,10 +528,14 @@ subtest_main(int argc, char *argv[], bool close_test) testutil_check(wiredtiger_open(opts->home, &event_handler, config, &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create( - session, "table:subtest", "key_format=i,value_format=iiiS,columns=(id,v0,v1,v2,big)")); + testutil_check(__wt_snprintf(tableconf, sizeof(tableconf), + "key_format=%s,value_format=iiiS,columns=(id,v0,v1,v2,big)", + opts->table_type == TABLE_ROW ? "i" : "r")); + testutil_check(session->create(session, "table:subtest", tableconf)); - testutil_check(session->create(session, "table:subtest2", "key_format=i,value_format=i")); + testutil_check(__wt_snprintf(tableconf, sizeof(tableconf), "key_format=%s,value_format=i", + opts->table_type == TABLE_ROW ? "i" : "r")); + testutil_check(session->create(session, "table:subtest2", tableconf)); testutil_check(session->create(session, "index:subtest:v0", "columns=(v0)")); testutil_check(session->create(session, "index:subtest:v1", "columns=(v1)")); @@ -579,11 +598,17 @@ subtest_populate(TEST_OPTS *opts, bool close_test) generate_key(i, &key); generate_value(rndint, i, bigref, &v0, &v1, &v2, &big); CHECK(session->begin_transaction(session, NULL), false); - maincur->set_key(maincur, key); + if (opts->table_type == TABLE_ROW) + maincur->set_key(maincur, key); + else + maincur->set_key(maincur, (uint64_t)key); maincur->set_value(maincur, v0, v1, v2, big); CHECK(maincur->insert(maincur), false); - maincur2->set_key(maincur2, key); + if (opts->table_type == TABLE_ROW) + maincur2->set_key(maincur2, key); + else + maincur2->set_key(maincur2, (uint64_t)key); maincur2->set_value(maincur2, rndint); CHECK(maincur2->insert(maincur2), false); CHECK(session->commit_transaction(session, NULL), false); @@ -641,6 +666,7 @@ main(int argc, char *argv[]) opts = &_opts; memset(opts, 0, sizeof(*opts)); + opts->table_type = TABLE_ROW; debugger = NULL; testutil_check(testutil_parse_opts(argc, argv, opts)); @@ -648,6 +674,8 @@ main(int argc, char *argv[]) argv += __wt_optind; if (opts->nrecords == 0) opts->nrecords = 50000; + if (opts->table_type == TABLE_FIX) + testutil_die(ENOTSUP, "Fixed-length column store not supported"); while (argc > 0) { if (strcmp(argv[0], "subtest") == 0) { diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/smoke.sh b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/smoke.sh new file mode 100755 index 00000000000..e7c8f15f5a6 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/smoke.sh @@ -0,0 +1,29 @@ +#! /bin/sh + +set -e + +# Smoke-test wt2909_checkpoint_integrity as part of running "make check". + +# The cmake build passes in the builddir with -b; it should be passed through. +builddir_arg= +while getopts ":b:" opt; do + case $opt in + b) builddir_arg="-b $OPTARG" ;; + esac +done +shift $(( OPTIND - 1 )) + +if [ -n "$1" ] +then + # If the test binary is passed in manually. + test_bin=$1 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_wt2909_checkpoint_integrity +fi + +$TEST_WRAPPER $test_bin $builddir_arg -t r +$TEST_WRAPPER $test_bin $builddir_arg -t c diff --git a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c index 20d5d7478b6..6e02ee125c6 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c @@ -76,10 +76,11 @@ main(int argc, char *argv[]) WT_MODIFY modify_entry; WT_SESSION *session, *session2; uint64_t i, j, offset; - char *large_doc; + char *large_doc, tableconf[128]; opts = &_opts; memset(opts, 0, sizeof(*opts)); + opts->table_type = TABLE_ROW; testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); @@ -87,12 +88,14 @@ main(int argc, char *argv[]) "create,cache_size=1G,statistics_log=(json,wait=1)", &opts->conn)); testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, uri, - "key_format=Q,value_format=u,leaf_item_max=64M,leaf_page_max=32k,memory_page_max=1M")); + testutil_check(__wt_snprintf(tableconf, sizeof(tableconf), + "key_format=%s,value_format=u,leaf_item_max=64M,leaf_page_max=32k,memory_page_max=1M", + opts->table_type == TABLE_ROW ? "Q" : "r")); + testutil_check(session->create(session, uri, tableconf)); testutil_check(session->open_cursor(session, uri, NULL, NULL, &c)); - /* Value is initialized with 'v' and has not significance to it. */ + /* Value is initialized with 'v' and has no significance to it. */ large_doc = dmalloc(DATASIZE); memset(large_doc, 'v', DATASIZE); value.data = large_doc; @@ -100,7 +103,7 @@ main(int argc, char *argv[]) /* Insert records. */ for (i = 0; i < NUM_DOCS; i++) { - c->set_key(c, i); + c->set_key(c, i + 1); c->set_value(c, &value); testutil_check(c->insert(c)); } @@ -124,7 +127,7 @@ main(int argc, char *argv[]) for (i = 0; i < NUM_DOCS; i++) { /* Position the cursor. */ testutil_check(session2->begin_transaction(session2, "isolation=snapshot")); - c->set_key(c, i); + c->set_key(c, i + 1); modify_entry.data.data = "abcdefghijklmnopqrstuvwxyz"; modify_entry.data.size = strlen(modify_entry.data.data); modify_entry.offset = offset; diff --git a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/smoke.sh b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/smoke.sh new file mode 100755 index 00000000000..e8cfe61081f --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/smoke.sh @@ -0,0 +1,20 @@ +#! /bin/sh + +set -e + +# Smoke-test wt4105_large_doc_small_upd as part of running "make check". + +if [ -n "$1" ] +then + # If the test binary is passed in manually. + test_bin=$1 +else + # If $top_builddir/$top_srcdir aren't set, default to building in build_posix + # and running in test/csuite. + top_builddir=${top_builddir:-../../build_posix} + top_srcdir=${top_srcdir:-../..} + test_bin=$top_builddir/test/csuite/test_wt4105_large_doc_small_upd +fi + +$TEST_WRAPPER $test_bin -t r +$TEST_WRAPPER $test_bin -t c diff --git a/src/third_party/wiredtiger/test/ctest_helpers.cmake b/src/third_party/wiredtiger/test/ctest_helpers.cmake index 622aea84160..1a8a7b4152e 100644 --- a/src/third_party/wiredtiger/test/ctest_helpers.cmake +++ b/src/third_party/wiredtiger/test/ctest_helpers.cmake @@ -229,6 +229,13 @@ macro(define_c_test) if(NOT "${C_TEST_FLAGS}" STREQUAL "") list(APPEND additional_executable_args FLAGS ${C_TEST_FLAGS}) endif() + set(exec_wrapper) + if(WT_WIN) + # This is a workaround to run our csuite tests under Windows using CTest. When executing a test, + # CTests by-passes the shell and directly executes the test as a child process. In doing so CTest executes the binary with forward-slash paths. + # Which while technically valid breaks assumptions in our testing utilities. Wrap the execution in powershell to avoid this. + set(exec_wrapper "powershell.exe") + endif() if (C_TEST_SMOKE) # csuite test comes with a smoke execution wrapper. create_test_executable(${C_TEST_TARGET} @@ -238,7 +245,7 @@ macro(define_c_test) ${additional_executable_args} ) add_test(NAME ${C_TEST_TARGET} - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME}/smoke.sh ${C_TEST_ARGUMENTS} $<TARGET_FILE:${C_TEST_TARGET}> + COMMAND ${exec_wrapper} ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME}/smoke.sh ${C_TEST_ARGUMENTS} $<TARGET_FILE:${C_TEST_TARGET}> WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} ) else() @@ -253,13 +260,6 @@ macro(define_c_test) # Ensure each DB home directory is run under the tests working directory. set(command_args -h ${wt_test_home_dir}) list(APPEND command_args ${C_TEST_ARGUMENTS}) - set(exec_wrapper) - if(WT_WIN) - # This is a workaround to run our csuite tests under Windows using CTest. When executing a test, - # CTests by-passes the shell and directly executes the test as a child process. In doing so CTest executes the binary with forward-slash paths. - # Which while technically valid breaks assumptions in our testing utilities. Wrap the execution in powershell to avoid this. - set(exec_wrapper "powershell.exe") - endif() add_test(NAME ${C_TEST_TARGET} COMMAND ${exec_wrapper} $<TARGET_FILE:${C_TEST_TARGET}> ${command_args} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 41362d91097..5bb3fd50bc2 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -2061,6 +2061,14 @@ tasks: set -o verbose env CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:/opt/java/jdk11/bin:$PATH sh s_release `date +%Y%m%d` + - name: doc-compile + tags: ["pull_request"] + run_on: + - ubuntu2004-test + commands: + - func: "get project" + - func: "compile wiredtiger docs" + - name: doc-update patchable: false commands: @@ -2886,7 +2894,7 @@ buildvariants: tasks: - name: package -- name: doc-update +- name: documentation-update display_name: "~ Documentation update" batchtime: 1440 # 1 day run_on: diff --git a/src/third_party/wiredtiger/test/suite/test_base01.py b/src/third_party/wiredtiger/test/suite/test_base01.py index 4bb5e826c5d..2b150c1f049 100644 --- a/src/third_party/wiredtiger/test/suite/test_base01.py +++ b/src/third_party/wiredtiger/test/suite/test_base01.py @@ -27,6 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_base01.py # Basic operations @@ -37,12 +38,19 @@ class test_base01(wttest.WiredTigerTestCase): table_name1 = 'test_base01a.wt' table_name2 = 'test_base01b.wt' + key_format_values = [ + ('column', dict(key_format='r')), + ('string_row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + def create_table(self, tablename): + format = 'key_format={},value_format=S'.format(self.key_format) extra_params = ',allocation_size=512,' +\ 'internal_page_max=16384,leaf_page_max=131072' self.pr('create_table') - self.session.create('table:' + tablename, - 'key_format=S,value_format=S' + extra_params) + self.session.create('table:' + tablename, format + extra_params) def cursor_s(self, tablename, key): cursor = self.session.open_cursor('table:' + tablename, None, None) @@ -72,9 +80,11 @@ class test_base01(wttest.WiredTigerTestCase): """ Create a table, look for a nonexistent key """ + somekey = 'somekey' if self.key_format == 'S' else 12345 + self.create_table(self.table_name1) self.pr('creating cursor') - cursor = self.cursor_s(self.table_name1, 'somekey') + cursor = self.cursor_s(self.table_name1, somekey) self.pr('search') ret = cursor.search() self.assertTrue(ret == wiredtiger.WT_NOTFOUND) @@ -85,15 +95,17 @@ class test_base01(wttest.WiredTigerTestCase): """ Create a table, add a key, get it back """ + key1 = 'key1' if self.key_format == 'S' else 42 + self.create_table(self.table_name2) self.pr('insert') - inscursor = self.cursor_ss(self.table_name2, 'key1', 'value1') + inscursor = self.cursor_ss(self.table_name2, key1, 'value1') inscursor.insert() inscursor.close self.pr('search') - getcursor = self.cursor_s(self.table_name2, 'key1') + getcursor = self.cursor_s(self.table_name2, key1) ret = getcursor.search() self.assertTrue(ret == 0) self.assertEqual(getcursor.get_value(), 'value1') diff --git a/src/third_party/wiredtiger/test/suite/test_bug004.py b/src/third_party/wiredtiger/test/suite/test_bug004.py index 8af470d5161..aedf35de731 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug004.py +++ b/src/third_party/wiredtiger/test/suite/test_bug004.py @@ -35,6 +35,7 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet, simple_key, simple_value +from wtscenario import make_scenarios # Check to make sure we see the right versions of overflow keys and values # when they are deleted in reconciliation without having been instantiated @@ -44,17 +45,31 @@ class test_bug004(wttest.WiredTigerTestCase): uri = 'file:test_ovfl_key' # Use a small page size because we want to create overflow items - config = 'allocation_size=512,' +\ - 'leaf_page_max=512,value_format=S,key_format=S' + config = 'allocation_size=512,leaf_page_max=512,' nentries = 30 + key_format_values = [ + ('column', dict(key_format='r')), + ('string_row', dict(key_format='S')), + ] + + scenarios = make_scenarios(key_format_values) + + def make_key(self, c1, i): + if self.key_format == 'S': + return simple_key(c1, i) + 'abcdef' * 100 + else: + return simple_key(c1, i) * 1000 + 551 + def test_bug004(self): # Create the object, fill with overflow keys and values. - self.session.create(self.uri, self.config) + format = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(self.uri, self.config + format) + c1 = self.session.open_cursor(self.uri, None) for i in range(1, self.nentries): - c1[simple_key(c1, i) + 'abcdef' * 100] = \ + c1[self.make_key(c1, i)] = \ simple_value(c1, i) + 'abcdef' * 100 c1.close() @@ -73,9 +88,9 @@ class test_bug004(wttest.WiredTigerTestCase): # currently do -- that's unlikely to change, but is a problem for the # test going forward.) c1 = self.session.open_cursor(self.uri, None) - c1.set_key(simple_key(c1, self.nentries - 5) + 'abcdef' * 100) + c1.set_key(self.make_key(c1, self.nentries - 5)) c2 = self.session.open_cursor(self.uri, None) - c2.set_key(simple_key(c2, self.nentries + 5) + 'abcdef' * 100) + c2.set_key(self.make_key(c2, self.nentries + 5)) self.session.truncate(None, c1, c2, None) c1.close() c2.close() @@ -85,12 +100,12 @@ class test_bug004(wttest.WiredTigerTestCase): # Use the snapshot cursor to retrieve the old key/value pairs c1 = tmp_session.open_cursor(self.uri, None) - c1.set_key(simple_key(c1, 1) + 'abcdef' * 100) + c1.set_key(self.make_key(c1, 1)) c1.search() for i in range(2, self.nentries): c1.next() self.assertEquals( - c1.get_key(), simple_key(c1, i) + 'abcdef' * 100) + c1.get_key(), self.make_key(c1, i)) self.assertEquals( c1.get_value(), simple_value(c1, i) + 'abcdef' * 100) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py index cf02983200e..b1da367d99a 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py @@ -39,15 +39,22 @@ from wtscenario import make_scenarios # Run background checkpoints repeatedly while doing inserts and other # operations in another thread class test_checkpoint02(wttest.WiredTigerTestCase): - scenarios = make_scenarios([ - ('table-100', dict(uri='table:test',fmt='L',dsize=100,nops=50000,nthreads=10)), - ('table-10', dict(uri='table:test',fmt='L',dsize=10,nops=50000,nthreads=30)) - ]) + key_format_values = [ + ('column', dict(key_format='r')), + ('u32_row', dict(key_format='L')), + ] + + size_values = [ + ('table-100', dict(uri='table:test',dsize=100,nops=50000,nthreads=10)), + ('table-10', dict(uri='table:test',dsize=10,nops=50000,nthreads=30)) + ] + + scenarios = make_scenarios(key_format_values, size_values) def test_checkpoint02(self): done = threading.Event() self.session.create(self.uri, - "key_format=" + self.fmt + ",value_format=S") + "key_format=" + self.key_format + ",value_format=S") ckpt = checkpoint_thread(self.conn, done) work_queue = queue.Queue() opthreads = [] @@ -59,11 +66,11 @@ class test_checkpoint02(wttest.WiredTigerTestCase): my_data = 'a' * self.dsize for i in range(self.nops): if i % 191 == 0 and i != 0: - work_queue.put_nowait(('b', i, my_data)) - work_queue.put_nowait(('i', i, my_data)) + work_queue.put_nowait(('b', i + 1, my_data)) + work_queue.put_nowait(('i', i + 1, my_data)) for i in range(self.nthreads): - t = op_thread(self.conn, uris, self.fmt, work_queue, done) + t = op_thread(self.conn, uris, self.key_format, work_queue, done) opthreads.append(t) t.start() except: @@ -81,7 +88,7 @@ class test_checkpoint02(wttest.WiredTigerTestCase): # Create a cursor - ensure all items have been put. cursor = self.session.open_cursor(self.uri, None, None) - i = 0 + i = 1 while True: nextret = cursor.next() if nextret != 0: @@ -92,7 +99,7 @@ class test_checkpoint02(wttest.WiredTigerTestCase): self.assertEqual(value, my_data) i += 1 - self.assertEqual(i, self.nops) + self.assertEqual(i, self.nops + 1) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py index 10f6138c2c0..106aaeaa00d 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py @@ -45,6 +45,13 @@ class test_checkpoint03(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot, ' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') val = stat_cursor[stat][2] @@ -53,7 +60,7 @@ class test_checkpoint03(wttest.WiredTigerTestCase, suite_subprocess): def test_checkpoint_writes_to_hs(self): # Create a basic table. - self.session.create(self.uri, 'key_format=i,value_format=i') + self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) self.session.begin_transaction() self.conn.set_timestamp('oldest_timestamp=1') @@ -96,7 +103,7 @@ class test_checkpoint03(wttest.WiredTigerTestCase, suite_subprocess): # Open a new connection and validate that we see the latest update as part of the datafile. conn2 = self.setUpConnectionOpen('.') session2 = self.setUpSessionOpen(conn2) - session2.create(self.uri, 'key_format=i,value_format=i') + session2.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) cur2 = session2.open_cursor(self.uri) cur2.set_key(1) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py index 06795a3cd27..91cd80667c6 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py @@ -28,6 +28,7 @@ import time import wiredtiger, wttest +from wtscenario import make_scenarios # test_checkpoint06.py # Verify that we rollback the truncation that is committed after stable @@ -36,16 +37,23 @@ class test_checkpoint06(wttest.WiredTigerTestCase): conn_config = 'create,cache_size=50MB' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + def test_rollback_truncation_in_checkpoint(self): self.uri = 'table:ckpt06' - self.session.create(self.uri, 'key_format=i,value_format=S') + self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) value = "abcdefghijklmnopqrstuvwxyz" * 3 self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() # Setup: Insert some data - for i in range(10000): + for i in range(1, 10001): cursor[i] = value self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2)) @@ -70,5 +78,5 @@ class test_checkpoint06(wttest.WiredTigerTestCase): # Verify the truncation is rolled back. cursor = self.session.open_cursor(self.uri) - for i in range(1000): + for i in range(1, 1001): self.assertEqual(cursor[i], value) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare15.py b/src/third_party/wiredtiger/test/suite/test_prepare15.py index b4db822caf6..1f8fb825042 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare15.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare15.py @@ -33,7 +33,7 @@ from wtscenario import make_scenarios # test_prepare15.py # Test that the prepare transaction rollback removes the on-disk key # or replace it with history store and commit retains the changes when -# both insert and remove operations are from the same transaction. +# both insert and remove operations are from the same prepared transaction. class test_prepare15(wttest.WiredTigerTestCase): in_memory_values = [ ('no_inmem', dict(in_memory=False)), @@ -60,10 +60,10 @@ class test_prepare15(wttest.WiredTigerTestCase): config += ',in_memory=false' return config - def test_prepare_restore_hs_update(self): + def test_prepare_hs_update_and_tombstone(self): # Create a table without logging. uri = "table:prepare15" - create_config = 'allocation_size=512,key_format=S,value_format=S' + create_config = 'key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. @@ -71,16 +71,16 @@ class test_prepare15(wttest.WiredTigerTestCase): ',stable_timestamp=' + self.timestamp_str(10)) valuea = 'a' - valueb = 'a' + valueb = 'b' # Perform an update and remove. cursor = self.session.open_cursor(uri) self.session.begin_transaction() - cursor[str(0)] = valuea + cursor[1] = valuea self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) self.session.begin_transaction() - cursor.set_key(str(0)) + cursor.set_key(1) cursor.remove() self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) cursor.close() @@ -89,8 +89,8 @@ class test_prepare15(wttest.WiredTigerTestCase): s = self.conn.open_session() cursor = s.open_cursor(uri) s.begin_transaction() - cursor[str(0)] = valueb - cursor.set_key(str(0)) + cursor[1] = valueb + cursor.set_key(1) cursor.remove() cursor.close() s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) @@ -100,19 +100,17 @@ class test_prepare15(wttest.WiredTigerTestCase): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction('ignore_prepare = true') - evict_cursor.set_key(str(0)) + evict_cursor.set_key(1) self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() if self.commit: - # Commit the prepared transaction s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50)) s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(60)) s.commit_transaction() else: - # Rollback the prepared transaction s.rollback_transaction() # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. @@ -120,7 +118,7 @@ class test_prepare15(wttest.WiredTigerTestCase): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction() - evict_cursor.set_key(str(0)) + evict_cursor.set_key(1) self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() @@ -128,66 +126,146 @@ class test_prepare15(wttest.WiredTigerTestCase): self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) cursor2 = self.session.open_cursor(uri) - cursor2.set_key(str(0)) + cursor2.set_key(1) self.assertEquals(cursor2.search(), 0) self.assertEqual(cursor2.get_value(), valuea) self.session.commit_transaction() - def test_prepare_not_found(self): + def test_prepare_hs_update(self): # Create a table without logging. uri = "table:prepare15" - create_config = 'allocation_size=512,key_format=S,value_format=S' + create_config = 'key_format={},value_format=S'.format(self.key_format) self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value = 'a' + valuea = 'a' + valueb = 'b' + valuec = 'c' + + # Perform an update. + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[1] = valuea + cursor.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) # Perform an update and remove. s = self.conn.open_session() cursor = s.open_cursor(uri) s.begin_transaction() - cursor[str(0)] = value - cursor.set_key(str(0)) + cursor[1] = valueb + cursor.set_key(1) cursor.remove() cursor.close() - s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(40)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search for the key so we position our cursor on the page that we want to evict. - self.session.begin_transaction("ignore_prepare = true") - evict_cursor.set_key(str(0)) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + self.session.begin_transaction('ignore_prepare = true') + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), 0) + self.assertEqual(evict_cursor.get_value(), valuea) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() if self.commit: - # Commit the prepared transaction - s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) - s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(40)) + s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(50)) + s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(60)) s.commit_transaction() else: - # Rollback the prepared transaction s.rollback_transaction() + # The history store update should be visible. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) + cursor2 = self.session.open_cursor(uri) + cursor2.set_key(1) + self.assertEquals(cursor2.search(), 0) + self.assertEqual(cursor2.get_value(), valuea) + self.session.commit_transaction() + + # Pin oldest and stable timestamps to 60. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(70) + + ',stable_timestamp=' + self.timestamp_str(70)) + + # Perform an additional update. + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[1] = valuec + cursor.close() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(80)) + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction() - evict_cursor.set_key(str(0)) + evict_cursor.set_key(1) + self.assertEquals(evict_cursor.search(), 0) + self.assertEqual(evict_cursor.get_value(), valuec) + evict_cursor.reset() + evict_cursor.close() + self.session.commit_transaction() + + # The history store update should not be visible when the prepared transaction commits + # due to global visibility of the stop timestamp. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(70)) + cursor2 = self.session.open_cursor(uri) + cursor2.set_key(1) + if self.commit: + self.assertEquals(cursor2.search(), WT_NOTFOUND) + else: + self.assertEquals(cursor2.search(), 0) + self.assertEqual(cursor2.get_value(), valuea) + self.session.commit_transaction() + + def test_prepare_no_hs(self): + # Create a table without logging. + uri = "table:prepare15" + create_config = 'key_format={},value_format=S'.format(self.key_format) + self.session.create(uri, create_config) + + # Pin oldest and stable timestamps to 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + value = 'a' + + # Perform an update and remove. + s = self.conn.open_session() + cursor = s.open_cursor(uri) + s.begin_transaction() + cursor[1] = value + cursor.set_key(1) + cursor.remove() + cursor.close() + s.prepare_transaction('prepare_timestamp=' + self.timestamp_str(20)) + + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + + # Search for the key so we position our cursor on the page that we want to evict. + self.session.begin_transaction("ignore_prepare = true") + evict_cursor.set_key(1) self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() + if self.commit: + s.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) + s.timestamp_transaction('durable_timestamp=' + self.timestamp_str(40)) + s.commit_transaction() + else: + s.rollback_transaction() + self.session.begin_transaction() cursor2 = self.session.open_cursor(uri) - cursor2.set_key(str(0)) + cursor2.set_key(1) self.assertEquals(cursor2.search(), WT_NOTFOUND) self.session.commit_transaction() |