diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2019-12-31 04:23:38 +0000 |
|---|---|---|
| committer | evergreen <evergreen@mongodb.com> | 2019-12-31 04:23:38 +0000 |
| commit | 8e2c33cdf3552a7ba3a96fef162a3463f16a33eb (patch) | |
| tree | 2aef5726ac1727fa676aca7414fa084c69e7641c /src/third_party/wiredtiger | |
| parent | b986b5bbc511af8cdeb55abb75d2a67c081ca35e (diff) | |
| download | mongo-8e2c33cdf3552a7ba3a96fef162a3463f16a33eb.tar.gz | |
Import wiredtiger: 61bf0547c6edfc6cd7c633c80f1c76f3e33e5aec from branch mongodb-4.4
ref: 77f9aebf23..61bf0547c6
for: 4.3.3
WT-4919 Add debug mode which allocates exact amount of memory in __wt_realloc_def
WT-4921 Add debug mode option that slows checkpoint creation
WT-4997 Migrate Jenkins “wiredtiger-test-format-stress” job to Evergreen
WT-4998 Migrate Jenkins “wiredtiger-test-format-stress-ppc” job to Evergreen
WT-5000 Migrate Jenkins “wiredtiger-test-format-stress-sanitizer” job to Evergreen
WT-5004 Migrate Jenkins “wiredtiger-test-checkpoint-stress” job to Evergreen
WT-5005 Migrate Jenkins “wiredtiger-test-recovery-stress” job to Evergreen
WT-5090 Log a message when rolling a thread back because it holds the oldest timestamp
WT-5112 Handling goto labels with multiple words in s_goto.py
WT-5312 Add thread doing random cursor operations to format's cache size calculation
WT-5324 WiredTiger API error handling fixes
WT-5327 Backup cursor API can fail to restore WT_SESSION name information
WT-5330 Read-only transactions are being rolled back in MongoDB
WT-5360 Add a template for building a glossary in developer docs
WT-5365 Remove format's long-running transaction support
Diffstat (limited to 'src/third_party/wiredtiger')
55 files changed, 941 insertions, 577 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index f4d0d31dab0..ab5f3ab49cc 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -493,7 +493,6 @@ t = env.Program("t_format", "test/format/bulk.c", "test/format/compact.c", "test/format/config.c", - "test/format/lrt.c", "test/format/ops.c", "test/format/rebalance.c", "test/format/random.c", diff --git a/src/third_party/wiredtiger/bench/wtperf/split_heavy.wtperf b/src/third_party/wiredtiger/bench/wtperf/split_heavy.wtperf new file mode 100644 index 00000000000..4f11340c095 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/split_heavy.wtperf @@ -0,0 +1,13 @@ +conn_config="cache_size=100MB,log=(enabled=false),statistics=[fast],statistics_log=(wait=5,json=false)" +table_config="leaf_page_max=8k,internal_page_max=8k,leaf_item_max=1433,internal_item_max=3100,type=file,memory_page_max=1MB,split_deepen_min_child=100" +table_count=3 +icount=50000 +populate_threads=1 +random_range=100000000 +checkpoint_threads=0 +report_interval=5 +run_time=10 +threads=((count=20,inserts=1)) +value_sz=200 +key_sz=64 +reopen_connection=false
\ No newline at end of file diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 697d59c8dcd..fb1df282cdb 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -401,7 +401,7 @@ worker_async(void *arg) break; goto op_err; default: - op_err: +op_err: lprintf(wtperf, ret, 0, "%s failed for: %s, range: %" PRIu64, op_name(op), key_buf, wtperf_value_range(wtperf)); goto err; /* can't happen */ @@ -831,7 +831,7 @@ worker(void *arg) if (ret == WT_NOTFOUND) break; - op_err: +op_err: if (ret == WT_ROLLBACK && (ops_per_txn != 0 || opts->log_like_table)) { /* * If we are running with explicit transactions configured and we hit a WT_ROLLBACK, diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 6f967a92118..d6280b7f0ec 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -501,10 +501,19 @@ connection_runtime_config = [ is not limited to not skewing newest, not favoring leaf pages, and modifying the eviction score mechanism.''', type='boolean'), + Config('realloc_exact', 'false', r''' + if true, reallocation of memory will only provide the exact + amount requested. This will help with spotting memory allocation + issues more easily.''', + type='boolean'), Config('rollback_error', '0', r''' return a WT_ROLLBACK error from a transaction operation about every Nth operation to simulate a collision''', min='0', max='10M'), + Config('slow_checkpoint', 'false', r''' + if true, slow down checkpoint creation by slowing down internal + page processing.''', + type='boolean'), Config('table_logging', 'false', r''' if true, write transaction related information to the log for all operations, even operations for tables with logging turned off. diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 3a5f70691a7..01fa0a7846b 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -1,12 +1,16 @@ # List of WiredTiger #defines that are "unused", but it's OK. API_CALL API_CALL_NOCONF +API_END API_SESSION_INIT +API_SESSION_POP +API_SESSION_PUSH FLD_MASK JOINABLE_CURSOR_CALL_CHECK LF_MASK LLONG_MAX LLONG_MIN +SESSION_API_PREPARE_CHECK TXN_API_CALL TXN_API_CALL_NOCONF TXN_API_END diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index 0218937fffc..54928fc03b4 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -34,6 +34,7 @@ __wt_stat_join_aggregate __wt_stat_join_clear_all __wt_stream_set_no_buffer __wt_try_readlock +__wt_txn_err_chk wiredtiger_calc_modify wiredtiger_config_parser_open wiredtiger_config_validate diff --git a/src/third_party/wiredtiger/dist/s_goto.py b/src/third_party/wiredtiger/dist/s_goto.py index 032084168cc..a7f35488603 100644 --- a/src/third_party/wiredtiger/dist/s_goto.py +++ b/src/third_party/wiredtiger/dist/s_goto.py @@ -6,7 +6,7 @@ import re, sys # 1. Zero or more whitespace characters. # 2. One or more lowercase ASCII characters. # 3. Colon character. -p = re.compile('^\s*[a-z]+:$') +p = re.compile('^\s*[a-z_]+:$') for line in sys.stdin: m = p.search(line) if m is not None: diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index ce4e98616c1..f2d431f97fa 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "77f9aebf23bb7fcb3911345741f20d67128f8da6", + "commit": "61bf0547c6edfc6cd7c633c80f1c76f3e33e5aec", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.4" diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c index 5ba9af81055..88c6fde49c3 100644 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ b/src/third_party/wiredtiger/src/async/async_op.c @@ -36,7 +36,7 @@ __async_set_key(WT_ASYNC_OP *asyncop, ...) c = &asyncop->c; va_start(ap, asyncop); - __wt_cursor_set_keyv(c, c->flags, ap); + WT_IGNORE_RET(__wt_cursor_set_keyv(c, c->flags, ap)); if (!WT_DATA_IN_ITEM(&c->key) && !WT_CURSOR_RECNO(c)) c->saved_err = __wt_buf_set(O2S((WT_ASYNC_OP_IMPL *)asyncop), &c->key, c->key.data, c->key.size); @@ -71,7 +71,7 @@ __async_set_value(WT_ASYNC_OP *asyncop, ...) c = &asyncop->c; va_start(ap, asyncop); - __wt_cursor_set_valuev(c, ap); + WT_IGNORE_RET(__wt_cursor_set_valuev(c, ap)); /* Copy the data, if it is pointing at data elsewhere. */ if (!WT_DATA_IN_ITEM(&c->value)) c->saved_err = diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index d4abddbd9dd..9502c70c218 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -57,7 +57,7 @@ __cursor_fix_append_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) cbt->v = 0; cbt->iface.value.data = &cbt->v; } else { - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) { cbt->v = 0; @@ -111,7 +111,7 @@ new_page: if (cbt->ins != NULL && cbt->recno != WT_INSERT_RECNO(cbt->ins)) cbt->ins = NULL; if (cbt->ins != NULL) - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); @@ -145,12 +145,12 @@ __cursor_var_append_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) for (;;) { cbt->ins = WT_SKIP_NEXT(cbt->ins); - new_page: +new_page: if (cbt->ins == NULL) return (WT_NOTFOUND); __cursor_set_recno(cbt, WT_INSERT_RECNO(cbt->ins)); - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) continue; @@ -209,8 +209,8 @@ __cursor_var_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) return (WT_NOTFOUND); __cursor_set_recno(cbt, cbt->recno + 1); - new_page: - restart_read: +new_page: +restart_read: /* Find the matching WT_COL slot. */ if ((cip = __col_var_search(cbt->ref, cbt->recno, &rle_start)) == NULL) return (WT_NOTFOUND); @@ -337,9 +337,9 @@ __cursor_row_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) if (cbt->ins != NULL) cbt->ins = WT_SKIP_NEXT(cbt->ins); - new_insert: +new_insert: cbt->iter_retry = WT_CBT_RETRY_INSERT; - restart_read_insert: +restart_read_insert: if ((ins = cbt->ins) != NULL) { WT_RET(__wt_txn_read(session, ins->upd, &upd)); if (upd == NULL) @@ -373,7 +373,7 @@ __cursor_row_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) cbt->iter_retry = WT_CBT_RETRY_PAGE; cbt->slot = cbt->row_iteration_slot / 2 - 1; - restart_read_page: +restart_read_page: rip = &page->pg_row[cbt->slot]; WT_RET(__wt_txn_read(session, WT_ROW_UPDATE(page, rip), &upd)); if (upd != NULL && upd->type == WT_UPDATE_TOMBSTONE) { diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 4f514db6495..049e9f48bc5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -198,7 +198,7 @@ __cursor_fix_append_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) cbt->iface.value.data = &cbt->v; } else { upd = NULL; - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) { cbt->v = 0; @@ -252,7 +252,7 @@ new_page: cbt->ins = NULL; upd = NULL; if (cbt->ins != NULL) - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) { cbt->v = __bit_getv_recno(cbt->ref, cbt->recno, btree->bitcnt); @@ -286,12 +286,12 @@ __cursor_var_append_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) for (;;) { WT_RET(__cursor_skip_prev(cbt)); - new_page: +new_page: if (cbt->ins == NULL) return (WT_NOTFOUND); __cursor_set_recno(cbt, WT_INSERT_RECNO(cbt->ins)); - restart_read: +restart_read: WT_RET(__wt_txn_read(session, cbt->ins->upd, &upd)); if (upd == NULL) continue; @@ -348,11 +348,11 @@ __cursor_var_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) for (;;) { __cursor_set_recno(cbt, cbt->recno - 1); - new_page: +new_page: if (cbt->recno < cbt->ref->ref_recno) return (WT_NOTFOUND); - restart_read: +restart_read: /* Find the matching WT_COL slot. */ if ((cip = __col_var_search(cbt->ref, cbt->recno, &rle_start)) == NULL) return (WT_NOTFOUND); @@ -488,9 +488,9 @@ __cursor_row_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) if (cbt->ins != NULL) WT_RET(__cursor_skip_prev(cbt)); - new_insert: +new_insert: cbt->iter_retry = WT_CBT_RETRY_INSERT; - restart_read_insert: +restart_read_insert: if ((ins = cbt->ins) != NULL) { WT_RET(__wt_txn_read(session, ins->upd, &upd)); if (upd == NULL) @@ -526,7 +526,7 @@ __cursor_row_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart) cbt->iter_retry = WT_CBT_RETRY_PAGE; cbt->slot = cbt->row_iteration_slot / 2 - 1; - restart_read_page: +restart_read_page: rip = &page->pg_row[cbt->slot]; WT_RET(__wt_txn_read(session, WT_ROW_UPDATE(page, rip), &upd)); if (upd != NULL && upd->type == WT_UPDATE_TOMBSTONE) { diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 687c44bc3fa..0f6b569b755 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -1142,7 +1142,7 @@ err: * subsequent iteration can succeed, we cannot return success.) */ if (0) { - search_notfound: +search_notfound: ret = WT_NOTFOUND; if (!iterating && !positioned && F_ISSET(cursor, WT_CURSTD_OVERWRITE)) ret = 0; diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 81adc9de7a1..ddf47a43c2e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -754,7 +754,7 @@ read: continue; } - skip_evict: +skip_evict: /* * If we read the page and are configured to not trash the cache, and no other thread * has already used the page, set the read generation so the page is evicted soon. diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index ea54d449576..d918b5d856c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -917,7 +917,7 @@ __slvg_col_range_overlap(WT_SESSION_IMPL *session, uint32_t a_slot, uint32_t b_s * Case #5: a_trk is a superset of b_trk and a_trk is more desirable -- discard b_trk. */ if (a_trk->trk_gen > b_trk->trk_gen) { - delete_b: +delete_b: /* * After page and overflow reconciliation, one (and only one) * page can reference an overflow record. But, if we split a @@ -1512,7 +1512,7 @@ __slvg_row_range_overlap(WT_SESSION_IMPL *session, uint32_t a_slot, uint32_t b_s * Case #5: a_trk is a superset of b_trk and a_trk is more desirable -- discard b_trk. */ if (a_trk->trk_gen > b_trk->trk_gen) { - delete_b: +delete_b: /* * After page and overflow reconciliation, one (and only one) * page can reference an overflow record. But, if we split a diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 037fb5cdcb6..f7297729967 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -271,6 +271,9 @@ __wt_sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) if (WT_PAGE_IS_INTERNAL(page)) { internal_bytes += page->memory_footprint; ++internal_pages; + /* Slow down checkpoints. */ + if (F_ISSET(conn, WT_CONN_DEBUG_SLOW_CKPT)) + __wt_sleep(0, 10000); } else { leaf_bytes += page->memory_footprint; ++leaf_pages; diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 7685547b351..175a4010db0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -423,7 +423,7 @@ __verify_tree(WT_SESSION_IMPL *session, WT_REF *ref, WT_CELL_UNPACK *addr_unpack goto recno_chk; case WT_PAGE_COL_VAR: recno = ref->ref_recno; - recno_chk: +recno_chk: if (recno != vs->record_total + 1) WT_RET_MSG(session, WT_ERROR, "page at %s has a starting record of %" PRIu64 " when the expected starting record is %" PRIu64, @@ -473,7 +473,7 @@ __verify_tree(WT_SESSION_IMPL *session, WT_REF *ref, WT_CELL_UNPACK *addr_unpack case WT_PAGE_COL_INT: case WT_PAGE_ROW_INT: if (addr_unpack->raw != WT_CELL_ADDR_INT) - celltype_err: +celltype_err: WT_RET_MSG(session, WT_ERROR, "page at %s, of type %s, is referenced in " "its parent by a cell of type %s", diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index 2d6654ebd43..29a89231cf3 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -557,7 +557,7 @@ __verify_dsk_row( current->size = prefix + unpack->size; } - key_compare: +key_compare: /* * Compare the current key against the last key. * @@ -770,7 +770,7 @@ __verify_dsk_col_var( goto match_err; } else if (cell_type == WT_CELL_VALUE && last.data != NULL && last.size == unpack->size && memcmp(last.data, unpack->data, last.size) == 0) - match_err: +match_err: WT_RET_VRFY(session, "data entries %" PRIu32 " and %" PRIu32 " on page at %s are identical and should " "have been run-length encoded", diff --git a/src/third_party/wiredtiger/src/btree/row_key.c b/src/third_party/wiredtiger/src/btree/row_key.c index d0524dfe5a3..caaca56eb88 100644 --- a/src/third_party/wiredtiger/src/btree/row_key.c +++ b/src/third_party/wiredtiger/src/btree/row_key.c @@ -168,7 +168,7 @@ __wt_row_leaf_key_work( #endif for (slot_offset = 0;;) { if (0) { - switch_and_jump: +switch_and_jump: /* Switching to a forward roll. */ WT_ASSERT(session, direction == BACKWARD); direction = FORWARD; diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index 26328dee4d6..aa427d93e44 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -533,7 +533,7 @@ leaf_only: * read-mostly workload. Check that case and get out fast. */ if (0) { - leaf_match: +leaf_match: cbt->compare = 0; cbt->slot = WT_ROW_SLOT(page, rip); return (0); diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 337dbb2e593..a24841fe67e 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -55,8 +55,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_compatibility_sub static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = { {"checkpoint_retention", "int", NULL, "min=0,max=1024", NULL, 0}, - {"eviction", "boolean", NULL, NULL, NULL, 0}, + {"eviction", "boolean", NULL, NULL, NULL, 0}, {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, {"rollback_error", "int", NULL, "min=0,max=10M", NULL, 0}, + {"slow_checkpoint", "boolean", NULL, NULL, NULL, 0}, {"table_logging", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_eviction_subconfigs[] = { @@ -105,7 +106,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"compatibility", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"error_prefix", "string", NULL, NULL, NULL, 0}, {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2}, {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0}, @@ -541,7 +542,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -608,7 +609,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -675,7 +676,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -740,7 +741,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 4}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -816,11 +817,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_overflow=(file_max=0),cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),compatibility=(release=)," "debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",file_manager=(close_handle_minimum=250,close_idle_time=30," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),error_prefix=,eviction=(threads_max=8," + "threads_min=1),eviction_checkpoint_target=1," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95," + "file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),io_capacity=(total=0),log=(archive=true," "os_cache_dirty_pct=0,prealloc=true,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4)," @@ -989,10 +991,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," "debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1019,10 +1021,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," "debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1048,10 +1050,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," @@ -1075,10 +1077,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),debug_mode=(checkpoint_retention=0,eviction=false," - "rollback_error=0,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 2b5f1fb7ae3..7013ca58238 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1796,9 +1796,21 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) else F_CLR(cache, WT_CACHE_EVICT_DEBUG_MODE); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.realloc_exact", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_DEBUG_REALLOC_EXACT); + else + F_CLR(conn, WT_CONN_DEBUG_REALLOC_EXACT); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.rollback_error", &cval)); txn_global->debug_rollback = (uint64_t)cval.val; + WT_RET(__wt_config_gets(session, cfg, "debug_mode.slow_checkpoint", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_DEBUG_SLOW_CKPT); + else + F_CLR(conn, WT_CONN_DEBUG_SLOW_CKPT); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.table_logging", &cval)); if (cval.val) FLD_SET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE); diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index cb7e11a64cc..be781118895 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -15,6 +15,9 @@ static int __backup_start( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, WT_CURSOR_BACKUP *, const char *[]); static int __backup_stop(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); +#define WT_CURSOR_BACKUP_CHECK_STOP(cursor) \ + WT_ERR(F_ISSET(((WT_CURSOR_BACKUP *)(cursor)), WT_CURBACKUP_FORCE_STOP) ? EINVAL : 0); + /* * __curbackup_next -- * WT_CURSOR->next method for the backup cursor type. diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 73690788dfb..9006690402b 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -136,11 +136,11 @@ format: } /* - * __curdump_set_key -- + * __curdump_set_keyv -- * WT_CURSOR->set_key for dump cursors. */ -static void -__curdump_set_key(WT_CURSOR *cursor, ...) +static int +__curdump_set_keyv(WT_CURSOR *cursor, va_list ap) { WT_CURSOR *child; WT_CURSOR_DUMP *cdump; @@ -150,18 +150,15 @@ __curdump_set_key(WT_CURSOR *cursor, ...) const uint8_t *up; const char *p; bool json; - va_list ap; cdump = (WT_CURSOR_DUMP *)cursor; child = cdump->child; CURSOR_API_CALL(cursor, session, set_key, NULL); - va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSTD_RAW)) p = va_arg(ap, WT_ITEM *)->data; else p = va_arg(ap, const char *); - va_end(ap); json = F_ISSET(cursor, WT_CURSTD_DUMP_JSON); if (json) @@ -188,7 +185,21 @@ err: cursor->saved_err = ret; F_CLR(cursor, WT_CURSTD_KEY_SET); } - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __curdump_set_key -- + * WT_CURSOR->set_key for dump cursors. + */ +static void +__curdump_set_key(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curdump_set_keyv(cursor, ap)); + va_end(ap); } /* @@ -238,29 +249,26 @@ err: } /* - * __curdump_set_value -- + * __curdump_set_valuev -- * WT_CURSOR->set_value for dump cursors. */ -static void -__curdump_set_value(WT_CURSOR *cursor, ...) +static int +__curdump_set_valuev(WT_CURSOR *cursor, va_list ap) { WT_CURSOR *child; WT_CURSOR_DUMP *cdump; WT_DECL_RET; WT_SESSION_IMPL *session; const char *p; - va_list ap; cdump = (WT_CURSOR_DUMP *)cursor; child = cdump->child; CURSOR_API_CALL(cursor, session, set_value, NULL); - va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSTD_RAW)) p = va_arg(ap, WT_ITEM *)->data; else p = va_arg(ap, const char *); - va_end(ap); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) WT_ERR(__wt_json_to_item(session, p, cursor->value_format, @@ -275,7 +283,21 @@ err: cursor->saved_err = ret; F_CLR(cursor, WT_CURSTD_VALUE_SET); } - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __curdump_set_value -- + * WT_CURSOR->set_value for dump cursors. + */ +static void +__curdump_set_value(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curdump_set_valuev(cursor, ap)); + va_end(ap); } /* Pass through a call to the underlying cursor. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index e675392939c..8de5878653e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -29,22 +29,38 @@ err: } /* - * __curindex_set_value -- + * __curindex_set_valuev -- * WT_CURSOR->set_value implementation for index cursors. */ -static void -__curindex_set_value(WT_CURSOR *cursor, ...) +static int +__curindex_set_valuev(WT_CURSOR *cursor, va_list ap) { WT_DECL_RET; WT_SESSION_IMPL *session; + WT_UNUSED(ap); + JOINABLE_CURSOR_API_CALL(cursor, session, set_value, NULL); WT_ERR_MSG(session, ENOTSUP, "WT_CURSOR.set_value not supported for index cursors"); err: cursor->saved_err = ret; F_CLR(cursor, WT_CURSTD_VALUE_SET); - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __curindex_set_value -- + * WT_CURSOR->set_value implementation for index cursors. + */ +static void +__curindex_set_value(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curindex_set_valuev(cursor, ap)); + va_end(ap); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index 4db139a8633..fedf1b07640 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -120,35 +120,46 @@ err: } /* - * __curstat_set_key -- + * __curstat_set_keyv -- * WT_CURSOR->set_key for statistics cursors. */ -static void -__curstat_set_key(WT_CURSOR *cursor, ...) +static int +__curstat_set_keyv(WT_CURSOR *cursor, va_list ap) { WT_CURSOR_STAT *cst; WT_DECL_RET; WT_ITEM *item; WT_SESSION_IMPL *session; - va_list ap; cst = (WT_CURSOR_STAT *)cursor; CURSOR_API_CALL(cursor, session, set_key, NULL); F_CLR(cursor, WT_CURSTD_KEY_SET); - va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSTD_RAW)) { item = va_arg(ap, WT_ITEM *); ret = __wt_struct_unpack(session, item->data, item->size, cursor->key_format, &cst->key); } else cst->key = va_arg(ap, int); - va_end(ap); if ((cursor->saved_err = ret) == 0) F_SET(cursor, WT_CURSTD_KEY_EXT); err: - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __curstat_set_key -- + * WT_CURSOR->set_key for statistics cursors. + */ +static void +__curstat_set_key(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curstat_set_keyv(cursor, ap)); + va_end(ap); } /* @@ -716,7 +727,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, c WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); if (0) { - config_err: +config_err: WT_ERR_MSG(session, EINVAL, "cursor's statistics configuration doesn't match the " "database statistics configuration"); diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 6140b453f86..7ff9273bf6d 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -253,7 +253,7 @@ __wt_cursor_set_key(WT_CURSOR *cursor, ...) va_list ap; va_start(ap, cursor); - __wt_cursor_set_keyv(cursor, cursor->flags, ap); + WT_IGNORE_RET(__wt_cursor_set_keyv(cursor, cursor->flags, ap)); va_end(ap); } @@ -377,7 +377,7 @@ err: * __wt_cursor_set_keyv -- * WT_CURSOR->set_key default implementation. */ -void +int __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) { WT_DECL_RET; @@ -453,7 +453,7 @@ err: } else __wt_free(session, tmp.mem); } - API_END(session, ret); + API_END_RET(session, ret); } /* @@ -516,7 +516,7 @@ __wt_cursor_set_value(WT_CURSOR *cursor, ...) va_list ap; va_start(ap, cursor); - __wt_cursor_set_valuev(cursor, ap); + WT_IGNORE_RET(__wt_cursor_set_valuev(cursor, ap)); va_end(ap); } @@ -524,7 +524,7 @@ __wt_cursor_set_value(WT_CURSOR *cursor, ...) * __wt_cursor_set_valuev -- * WT_CURSOR->set_value worker implementation. */ -void +int __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap) { WT_DECL_RET; @@ -588,7 +588,7 @@ err: __wt_free(session, tmp.mem); } - API_END(session, ret); + API_END_RET(session, ret); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 94acee0592e..a5052532542 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -16,8 +16,11 @@ static int __curtable_update(WT_CURSOR *cursor); WT_CURSOR **__cp; \ u_int __i; \ for (__i = 0, __cp = (ctable)->cg_cursors; __i < WT_COLGROUPS((ctable)->table); \ - __i++, __cp++) \ + __i++, __cp++) { \ WT_TRET((*__cp)->f(*__cp)); \ + if (ret != 0 && ret != WT_NOTFOUND) \ + goto err; \ + } \ } while (0) /* Cursor type for custom extractor callback. */ @@ -224,7 +227,7 @@ __wt_curtable_set_key(WT_CURSOR *cursor, ...) primary = *cp++; va_start(ap, cursor); - __wt_cursor_set_keyv(primary, cursor->flags, ap); + WT_IGNORE_RET(__wt_cursor_set_keyv(primary, cursor->flags, ap)); va_end(ap); if (!F_ISSET(primary, WT_CURSTD_KEY_SET)) @@ -240,11 +243,11 @@ __wt_curtable_set_key(WT_CURSOR *cursor, ...) } /* - * __wt_curtable_set_value -- + * __curtable_set_valuev -- * WT_CURSOR->set_value implementation for tables. */ -void -__wt_curtable_set_value(WT_CURSOR *cursor, ...) +static int +__curtable_set_valuev(WT_CURSOR *cursor, va_list ap) { WT_CURSOR **cp; WT_CURSOR_TABLE *ctable; @@ -252,12 +255,10 @@ __wt_curtable_set_value(WT_CURSOR *cursor, ...) WT_ITEM *item, *tmp; WT_SESSION_IMPL *session; u_int i; - va_list ap; ctable = (WT_CURSOR_TABLE *)cursor; JOINABLE_CURSOR_API_CALL(cursor, session, set_value, NULL); - va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSOR_RAW_OK | WT_CURSTD_DUMP_JSON)) { item = va_arg(ap, WT_ITEM *); cursor->value.data = item->data; @@ -292,7 +293,6 @@ __wt_curtable_set_value(WT_CURSOR *cursor, ...) } } } - va_end(ap); for (i = 0, cp = ctable->cg_cursors; i < WT_COLGROUPS(ctable->table); i++, cp++) if (ret == 0) @@ -303,7 +303,21 @@ __wt_curtable_set_value(WT_CURSOR *cursor, ...) } err: - API_END(session, ret); + API_END_RET(session, ret); +} + +/* + * __wt_curtable_set_value -- + * WT_CURSOR->set_value implementation for tables. + */ +void +__wt_curtable_set_value(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + WT_IGNORE_RET(__curtable_set_valuev(cursor, ap)); + va_end(ap); } /* diff --git a/src/third_party/wiredtiger/src/docs/devdoc-glossary.dox b/src/third_party/wiredtiger/src/docs/devdoc-glossary.dox new file mode 100644 index 00000000000..4cb762cbaa7 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/devdoc-glossary.dox @@ -0,0 +1,30 @@ +/*! @page devdoc-glossary Developer Documentation Glossary of Terms + +WiredTiger has a lot of domain specific nomenclature - this page attempts +to decode it. This is intended for those navigating the WiredTiger source +tree - it describes terms internal to the storage engine. + +@section devdoc-glossary-general General Terms Used in WiredTiger + +<table> +<caption id="general_terms">General Term Table</caption> +<tr><th>Term <th>Definition +<tr><td>Hello<td>The typical next word is World +</table> + +@section devdoc-glossary-checkpoint Terms Related to Checkpoints + +<table> +<caption id="checkpoint_terms">Checkpoint Term Table</caption> +<tr><th>Term <th>Definition +<tr><td>Hello<td>The typical next word is World +</table> +@section devdoc-glossary-transactions Terms Related to Transactions + +<table> +<caption id="transaction_terms">Transaction Term Table</caption> +<tr><th>Term <th>Definition +<tr><td>Hello<td>The typical next word is World +</table> + +*/ diff --git a/src/third_party/wiredtiger/src/docs/devdoc-index.dox b/src/third_party/wiredtiger/src/docs/devdoc-index.dox index d88826cb52d..f7707653207 100644 --- a/src/third_party/wiredtiger/src/docs/devdoc-index.dox +++ b/src/third_party/wiredtiger/src/docs/devdoc-index.dox @@ -1,5 +1,9 @@ /*! @page devdoc-index Developer Documentation +@subpage devdoc-glossary + +WiredTiger assigns specific meanings to certain words. Here we decode them. + @subpage devdoc-schema Most applications begin to make use of WiredTiger by creating a table (or other diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 0faaacc710c..3393bf36501 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -2300,6 +2300,10 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, bool readonly, d if (ret == WT_ROLLBACK) { --cache->evict_aggressive_score; WT_STAT_CONN_INCR(session, txn_fail_cache); + if (timer) + WT_IGNORE_RET(__wt_msg(session, + "Application thread returned a rollback error because " + "it was forced to evict and eviction is stuck")); } WT_ERR(ret); } diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 533f276b15c..95946dfbc65 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -32,24 +32,31 @@ #define WT_SINGLE_THREAD_CHECK_STOP(s) #endif +#define API_SESSION_PUSH(s, h, n, dh) \ + WT_DATA_HANDLE *__olddh = (s)->dhandle; \ + const char *__oldname = (s)->name; \ + (s)->dhandle = (dh); \ + (s)->name = (s)->lastop = #h "." #n +#define API_SESSION_POP(s) \ + (s)->dhandle = __olddh; \ + (s)->name = __oldname + /* Standard entry points to the API: declares/initializes local variables. */ -#define API_SESSION_INIT(s, h, n, dh) \ - WT_TRACK_OP_DECL; \ - WT_DATA_HANDLE *__olddh = (s)->dhandle; \ - const char *__oldname = (s)->name; \ - (s)->dhandle = (dh); \ - (s)->name = (s)->lastop = #h "." #n; \ - /* \ - * No code before this line, otherwise error handling won't be \ - * correct. \ - */ \ - WT_TRACK_OP_INIT(s); \ - WT_SINGLE_THREAD_CHECK_START(s); \ - __wt_op_timer_start(s); \ - WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ - /* Reset wait time if this isn't an API reentry. */ \ - if (__oldname == NULL) \ - (s)->cache_wait_us = 0; \ +#define API_SESSION_INIT(s, h, n, dh) \ + WT_TRACK_OP_DECL; \ + API_SESSION_PUSH(s, h, n, dh); \ + /* \ + * No code before this line, otherwise error handling won't be \ + * correct. \ + */ \ + WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ + WT_SINGLE_THREAD_CHECK_START(s); \ + WT_ERR(__wt_txn_err_chk(s)); \ + WT_TRACK_OP_INIT(s); \ + __wt_op_timer_start(s); \ + /* Reset wait time if this isn't an API reentry. */ \ + if (__oldname == NULL) \ + (s)->cache_wait_us = 0; \ __wt_verbose((s), WT_VERB_API, "%s", "CALL: " #h ":" #n) #define API_CALL_NOCONF(s, h, n, dh) \ @@ -75,8 +82,7 @@ * No code after this line, otherwise error handling \ * won't be correct. \ */ \ - (s)->dhandle = __olddh; \ - (s)->name = __oldname; \ + API_SESSION_POP(s); \ } \ } \ while (0) @@ -115,7 +121,7 @@ if (__autotxn) { \ if (F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT)) \ F_CLR(&(s)->txn, WT_TXN_AUTOCOMMIT); \ - else if ((ret) == 0 && !F_ISSET(&(s)->txn, WT_TXN_ERROR)) \ + else if ((ret) == 0) \ (ret) = __wt_txn_commit((s), NULL); \ else { \ if (retry) \ @@ -167,24 +173,33 @@ #define SESSION_API_CALL_PREPARE_ALLOWED(s, n, config, cfg) \ API_CALL(s, WT_SESSION, n, NULL, config, cfg) -#define SESSION_API_CALL(s, n, config, cfg) \ - API_CALL(s, WT_SESSION, n, NULL, config, cfg); \ - WT_ERR(__wt_txn_context_prepare_check((s))) +#define SESSION_API_PREPARE_CHECK(s, h, n) \ + do { \ + int __prepare_ret; \ + API_SESSION_PUSH(s, WT_SESSION, n, NULL); \ + __prepare_ret = __wt_txn_context_prepare_check(s); \ + API_SESSION_POP(s); \ + WT_RET(__prepare_ret); \ + } while (0) + +#define SESSION_API_CALL(s, n, config, cfg) \ + SESSION_API_PREPARE_CHECK(s, WT_SESSION, n); \ + API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define SESSION_API_CALL_NOCONF(s, n) API_CALL_NOCONF(s, WT_SESSION, n, NULL) #define SESSION_API_CALL_NOCONF_PREPARE_NOT_ALLOWED(s, n) \ - API_CALL_NOCONF(s, WT_SESSION, n, NULL); \ - WT_ERR(__wt_txn_context_prepare_check((s))) + SESSION_API_PREPARE_CHECK(s, WT_SESSION, n); \ + API_CALL_NOCONF(s, WT_SESSION, n, NULL) -#define SESSION_TXN_API_CALL(s, n, config, cfg) \ - TXN_API_CALL(s, WT_SESSION, n, NULL, config, cfg); \ - WT_ERR(__wt_txn_context_prepare_check((s))) +#define SESSION_TXN_API_CALL(s, n, config, cfg) \ + SESSION_API_PREPARE_CHECK(s, WT_SESSION, n); \ + TXN_API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define CURSOR_API_CALL(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ + SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ API_CALL_NOCONF(s, WT_CURSOR, n, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ - WT_ERR(__wt_txn_context_prepare_check((s))); \ if (F_ISSET(cur, WT_CURSTD_CACHED)) \ WT_ERR(__wt_cursor_cached(cur)) @@ -206,11 +221,10 @@ CURSOR_API_CALL_PREPARE_ALLOWED(cur, s, n, bt); \ JOINABLE_CURSOR_CALL_CHECK(cur) -#define CURSOR_REMOVE_API_CALL(cur, s, bt) \ - (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF( \ - s, WT_CURSOR, remove, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ - WT_ERR(__wt_txn_context_prepare_check((s))) +#define CURSOR_REMOVE_API_CALL(cur, s, bt) \ + (s) = (WT_SESSION_IMPL *)(cur)->session; \ + SESSION_API_PREPARE_CHECK(s, WT_CURSOR, remove); \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, remove, ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle) #define JOINABLE_CURSOR_REMOVE_API_CALL(cur, s, bt) \ CURSOR_REMOVE_API_CALL(cur, s, bt); \ @@ -218,16 +232,16 @@ #define CURSOR_UPDATE_API_CALL_BTREE(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ + SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ TXN_API_CALL_NOCONF(s, WT_CURSOR, n, ((WT_BTREE *)(bt))->dhandle); \ - WT_ERR(__wt_txn_context_prepare_check((s))); \ if (F_ISSET(S2C(s), WT_CONN_IN_MEMORY) && !F_ISSET((WT_BTREE *)(bt), WT_BTREE_IGNORE_CACHE) && \ __wt_cache_full(s)) \ WT_ERR(WT_CACHE_FULL); #define CURSOR_UPDATE_API_CALL(cur, s, n) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF(s, WT_CURSOR, n, NULL); \ - WT_ERR(__wt_txn_context_prepare_check((s))) + SESSION_API_PREPARE_CHECK(s, WT_CURSOR, n); \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, n, NULL) #define JOINABLE_CURSOR_UPDATE_API_CALL(cur, s, n) \ CURSOR_UPDATE_API_CALL(cur, s, n); \ diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index ab810ee86fb..f77712972b6 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -519,27 +519,29 @@ struct __wt_connection_impl { #define WT_CONN_CLOSING_TIMESTAMP 0x00000020u #define WT_CONN_COMPATIBILITY 0x00000040u #define WT_CONN_DATA_CORRUPTION 0x00000080u -#define WT_CONN_EVICTION_NO_LOOKASIDE 0x00000100u -#define WT_CONN_EVICTION_RUN 0x00000200u -#define WT_CONN_INCR_BACKUP 0x00000400u -#define WT_CONN_IN_MEMORY 0x00000800u -#define WT_CONN_LEAK_MEMORY 0x00001000u -#define WT_CONN_LOOKASIDE_OPEN 0x00002000u -#define WT_CONN_LSM_MERGE 0x00004000u -#define WT_CONN_OPTRACK 0x00008000u -#define WT_CONN_PANIC 0x00010000u -#define WT_CONN_READONLY 0x00020000u -#define WT_CONN_RECONFIGURING 0x00040000u -#define WT_CONN_RECOVERING 0x00080000u -#define WT_CONN_SALVAGE 0x00100000u -#define WT_CONN_SERVER_ASYNC 0x00200000u -#define WT_CONN_SERVER_CAPACITY 0x00400000u -#define WT_CONN_SERVER_CHECKPOINT 0x00800000u -#define WT_CONN_SERVER_LOG 0x01000000u -#define WT_CONN_SERVER_LSM 0x02000000u -#define WT_CONN_SERVER_STATISTICS 0x04000000u -#define WT_CONN_SERVER_SWEEP 0x08000000u -#define WT_CONN_WAS_BACKUP 0x10000000u +#define WT_CONN_DEBUG_REALLOC_EXACT 0x00000100u +#define WT_CONN_DEBUG_SLOW_CKPT 0x00000200u +#define WT_CONN_EVICTION_NO_LOOKASIDE 0x00000400u +#define WT_CONN_EVICTION_RUN 0x00000800u +#define WT_CONN_INCR_BACKUP 0x00001000u +#define WT_CONN_IN_MEMORY 0x00002000u +#define WT_CONN_LEAK_MEMORY 0x00004000u +#define WT_CONN_LOOKASIDE_OPEN 0x00008000u +#define WT_CONN_LSM_MERGE 0x00010000u +#define WT_CONN_OPTRACK 0x00020000u +#define WT_CONN_PANIC 0x00040000u +#define WT_CONN_READONLY 0x00080000u +#define WT_CONN_RECONFIGURING 0x00100000u +#define WT_CONN_RECOVERING 0x00200000u +#define WT_CONN_SALVAGE 0x00400000u +#define WT_CONN_SERVER_ASYNC 0x00800000u +#define WT_CONN_SERVER_CAPACITY 0x01000000u +#define WT_CONN_SERVER_CHECKPOINT 0x02000000u +#define WT_CONN_SERVER_LOG 0x04000000u +#define WT_CONN_SERVER_LSM 0x08000000u +#define WT_CONN_SERVER_STATISTICS 0x10000000u +#define WT_CONN_SERVER_SWEEP 0x20000000u +#define WT_CONN_WAS_BACKUP 0x40000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 39740cc2be4..45a4ac01a9f 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -52,7 +52,9 @@ struct __wt_cursor_backup { size_t next; /* Cursor position */ WT_FSTREAM *bfs; /* Backup file stream */ - uint32_t maxid; /* Maximum log file ID seen */ + +#define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)(cursor))->maxid) + uint32_t maxid; /* Maximum log file ID seen */ char **list; /* List of files to be copied. */ size_t list_allocated; @@ -86,9 +88,6 @@ struct __wt_cursor_backup { /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; }; -#define WT_CURSOR_BACKUP_CHECK_STOP(cursor) \ - WT_RET(F_ISSET(((WT_CURSOR_BACKUP *)(cursor)), WT_CURBACKUP_FORCE_STOP) ? EINVAL : 0); -#define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)(cursor))->maxid) struct __wt_cursor_btree { WT_CURSOR iface; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 5e530b8aaa1..f3710895938 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -551,6 +551,10 @@ extern int __wt_cursor_reopen_notsup(WT_CURSOR *cursor, bool check_only) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp, bool *valid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curstat_colgroup_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], @@ -1601,13 +1605,11 @@ extern void __wt_cursor_key_order_reset(WT_CURSOR_BTREE *cbt); extern void __wt_cursor_reopen(WT_CURSOR *cursor, WT_DATA_HANDLE *dhandle); extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...); extern void __wt_cursor_set_key_notsup(WT_CURSOR *cursor, ...); -extern void __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap); extern void __wt_cursor_set_notsup(WT_CURSOR *cursor); extern void __wt_cursor_set_raw_key(WT_CURSOR *cursor, WT_ITEM *key); extern void __wt_cursor_set_raw_value(WT_CURSOR *cursor, WT_ITEM *value); extern void __wt_cursor_set_value(WT_CURSOR *cursor, ...); extern void __wt_cursor_set_value_notsup(WT_CURSOR *cursor, ...); -extern void __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap); extern void __wt_curstat_cache_walk(WT_SESSION_IMPL *session); extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst); extern void __wt_curtable_set_key(WT_CURSOR *cursor, ...); @@ -1960,6 +1962,8 @@ static inline int __wt_txn_context_check(WT_SESSION_IMPL *session, bool requires WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_context_prepare_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_txn_err_chk(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_id_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_idle_cache_check(WT_SESSION_IMPL *session) diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index c303edc9488..7d5c99e7980 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -110,11 +110,14 @@ * Common case allocate-and-grow function. Starts by allocating the requested number of items * (at least 10), then doubles each time the list needs to grow. */ -#define __wt_realloc_def(session, sizep, number, addr) \ - (((number) * sizeof(**(addr)) <= *(sizep)) ? \ - 0 : \ - __wt_realloc( \ - session, sizep, WT_MAX(*(sizep)*2, WT_MAX(10, (number)) * sizeof(**(addr))), addr)) +#define __wt_realloc_def(session, sizep, number, addr) \ + (((number) * sizeof(**(addr)) <= *(sizep)) ? \ + 0 : \ + __wt_realloc(session, sizep, (F_ISSET(S2C(session), WT_CONN_DEBUG_REALLOC_EXACT)) ? \ + (number) * sizeof(**(addr)) : \ + WT_MAX(*(sizep)*2, WT_MAX(10, (number)) * sizeof(**(addr))), \ + addr)) + /* * Our internal free function clears the underlying address atomically so there is a smaller chance * of racing threads seeing intermediate results while a structure is being free'd. (That would be a diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index d739e78cf28..65780e73c9d 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -142,32 +142,6 @@ __wt_snprintf_len_incr(char *buf, size_t size, size_t *retsizep, const char *fmt } /* - * __wt_txn_context_prepare_check -- - * Return an error if the current transaction is in the prepare state. - */ -static inline int -__wt_txn_context_prepare_check(WT_SESSION_IMPL *session) -{ - if (F_ISSET(&session->txn, WT_TXN_PREPARE)) - WT_RET_MSG(session, EINVAL, "%s: not permitted in a prepared transaction", session->name); - return (0); -} - -/* - * __wt_txn_context_check -- - * Complain if a transaction is/isn't running. - */ -static inline int -__wt_txn_context_check(WT_SESSION_IMPL *session, bool requires_txn) -{ - if (requires_txn && !F_ISSET(&session->txn, WT_TXN_RUNNING)) - WT_RET_MSG(session, EINVAL, "%s: only permitted in a running transaction", session->name); - if (!requires_txn && F_ISSET(&session->txn, WT_TXN_RUNNING)) - WT_RET_MSG(session, EINVAL, "%s: not permitted in a running transaction", session->name); - return (0); -} - -/* * __wt_spin_backoff -- * Back off while spinning for a resource. This is used to avoid busy waiting loops that can * consume enough CPU to block real work being done. The algorithm spins a few times, then diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 757627dff97..332f3a3735c 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -35,6 +35,50 @@ __wt_ref_cas_state_int(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t old_state } /* + * __wt_txn_context_prepare_check -- + * Return an error if the current transaction is in the prepare state. + */ +static inline int +__wt_txn_context_prepare_check(WT_SESSION_IMPL *session) +{ + if (F_ISSET(&session->txn, WT_TXN_PREPARE)) + WT_RET_MSG(session, EINVAL, "not permitted in a prepared transaction"); + return (0); +} + +/* + * __wt_txn_context_check -- + * Complain if a transaction is/isn't running. + */ +static inline int +__wt_txn_context_check(WT_SESSION_IMPL *session, bool requires_txn) +{ + if (requires_txn && !F_ISSET(&session->txn, WT_TXN_RUNNING)) + WT_RET_MSG(session, EINVAL, "only permitted in a running transaction"); + if (!requires_txn && F_ISSET(&session->txn, WT_TXN_RUNNING)) + WT_RET_MSG(session, EINVAL, "not permitted in a running transaction"); + return (0); +} + +/* + * __wt_txn_err_chk -- + * Check the transaction hasn't already failed. + */ +static inline int +__wt_txn_err_chk(WT_SESSION_IMPL *session) +{ + /* Allow transaction rollback, but nothing else. */ + if (!F_ISSET(&(session->txn), WT_TXN_ERROR) || + strcmp(session->name, "rollback_transaction") != 0) + return (0); + +#ifdef HAVE_DIAGNOSTIC + WT_ASSERT(session, !F_ISSET(&(session->txn), WT_TXN_ERROR)); +#endif + WT_RET_MSG(session, EINVAL, "additional transaction operations attempted after error"); +} + +/* * __wt_txn_timestamp_flags -- * Set transaction related timestamp flags. */ diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index e8a872178dd..a50e617705b 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2206,14 +2206,19 @@ struct __wt_connection { * algorithms to change skew to force lookaside eviction to happen more aggressively. This * includes but is not limited to not skewing newest\, not favoring leaf pages\, and * modifying the eviction score mechanism., a boolean flag; default \c false.} - * @config{ rollback_error, return a WT_ROLLBACK error from a - * transaction operation about every Nth operation to simulate a collision., an integer - * between 0 and 10M; default \c 0.} - * @config{ table_logging, if - * true\, write transaction related information to the log for all operations\, even - * operations for tables with logging turned off. This setting introduces a log format - * change that may break older versions of WiredTiger. These operations are informational - * and skipped in recovery., a boolean flag; default \c false.} + * @config{ realloc_exact, if true\, reallocation of memory will only + * provide the exact amount requested. This will help with spotting memory allocation + * issues more easily., a boolean flag; default \c false.} + * @config{ + * rollback_error, return a WT_ROLLBACK error from a transaction operation about every Nth + * operation to simulate a collision., an integer between 0 and 10M; default \c 0.} + * @config{ slow_checkpoint, if true\, slow down checkpoint creation + * by slowing down internal page processing., a boolean flag; default \c false.} + * @config{ table_logging, if true\, write transaction related + * information to the log for all operations\, even operations for tables with logging + * turned off. This setting introduces a log format change that may break older versions of + * WiredTiger. These operations are informational and skipped in recovery., a boolean flag; + * default \c false.} * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; default empty.} * @config{eviction = (, eviction configuration options., a set of related configuration @@ -2807,14 +2812,18 @@ struct __wt_connection { * skew to force lookaside eviction to happen more aggressively. This includes but is not limited * to not skewing newest\, not favoring leaf pages\, and modifying the eviction score mechanism., a * boolean flag; default \c false.} - * @config{ rollback_error, return a - * WT_ROLLBACK error from a transaction operation about every Nth operation to simulate a - * collision., an integer between 0 and 10M; default \c 0.} - * @config{ - * table_logging, if true\, write transaction related information to the log for all operations\, - * even operations for tables with logging turned off. This setting introduces a log format change - * that may break older versions of WiredTiger. These operations are informational and skipped in - * recovery., a boolean flag; default \c false.} + * @config{ realloc_exact, if true\, + * reallocation of memory will only provide the exact amount requested. This will help with + * spotting memory allocation issues more easily., a boolean flag; default \c false.} + * @config{ rollback_error, return a WT_ROLLBACK error from a transaction + * operation about every Nth operation to simulate a collision., an integer between 0 and 10M; + * default \c 0.} + * @config{ slow_checkpoint, if true\, slow down checkpoint + * creation by slowing down internal page processing., a boolean flag; default \c false.} + * @config{ table_logging, if true\, write transaction related information to + * the log for all operations\, even operations for tables with logging turned off. This setting + * introduces a log format change that may break older versions of WiredTiger. These operations are + * informational and skipped in recovery., a boolean flag; default \c false.} * @config{ ),,} * @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c FILE_FLAG_NO_BUFFERING on Windows to * access files. Options are given as a list\, such as <code>"direct_io=[data]"</code>. Configuring diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index ca6b0b4adaa..4b9510d372a 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -731,7 +731,7 @@ __wt_rec_col_var( */ WT_ERR(__wt_dsk_cell_data_ref(session, WT_PAGE_COL_VAR, vpack, orig)); - record_loop: +record_loop: /* * Generate on-page entries: loop repeat records, looking for WT_INSERT entries matching the * record number. The WT_INSERT lists are in sorted order, so only need check the next one. diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 43360d702c5..dbef848c3f3 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -989,7 +989,7 @@ build: /* Update compression state. */ __rec_key_state_update(r, ovfl_key); - leaf_insert: +leaf_insert: /* Write any K/V pairs inserted into the page after this key. */ if ((ins = WT_SKIP_FIRST(WT_ROW_INSERT(page, rip))) != NULL) WT_ERR(__rec_row_leaf_insert(session, r, ins)); diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 098acb87c60..d956c9692d9 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -1643,10 +1643,13 @@ __session_commit_transaction(WT_SESSION *wt_session, const char *config) WT_ERR(__wt_txn_context_check(session, true)); - if (F_ISSET(txn, WT_TXN_ERROR) && txn->mod_count != 0) - WT_ERR_MSG(session, EINVAL, "failed transaction requires rollback%s%s", + /* Permit the commit if the transaction failed, but was read-only. */ + if (F_ISSET(txn, WT_TXN_ERROR) && txn->mod_count != 0) { + __wt_err(session, EINVAL, "failed transaction requires rollback%s%s", txn->rollback_reason == NULL ? "" : ": ", txn->rollback_reason == NULL ? "" : txn->rollback_reason); + ret = EINVAL; + } if (ret == 0) ret = __wt_txn_commit(session, cfg); @@ -1656,7 +1659,6 @@ __session_commit_transaction(WT_SESSION *wt_session, const char *config) } err: - WT_ASSERT(session, WT_SESSION_TXN_STATE(session)->id == WT_TXN_NONE); API_END_RET(session, ret); } @@ -1669,7 +1671,6 @@ __session_prepare_transaction(WT_SESSION *wt_session, const char *config) { WT_DECL_RET; WT_SESSION_IMPL *session; - WT_TXN *txn; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, prepare_transaction, config, cfg); @@ -1678,19 +1679,9 @@ __session_prepare_transaction(WT_SESSION *wt_session, const char *config) WT_ERR(__wt_txn_context_check(session, true)); - /* - * A failed transaction cannot be prepared, as it cannot guarantee a subsequent commit. - */ - txn = &session->txn; - if (F_ISSET(txn, WT_TXN_ERROR) && txn->mod_count != 0) - WT_ERR_MSG(session, EINVAL, "failed transaction requires rollback%s%s", - txn->rollback_reason == NULL ? "" : ": ", - txn->rollback_reason == NULL ? "" : txn->rollback_reason); - WT_ERR(__wt_txn_prepare(session, cfg)); err: - WT_ASSERT(session, WT_SESSION_TXN_STATE(session)->id == WT_TXN_NONE); API_END_RET(session, ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index e8e97b2df57..0fae8db558d 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -892,6 +892,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) prepare = F_ISSET(txn, WT_TXN_PREPARE); readonly = txn->mod_count == 0; + /* Permit the commit if the transaction failed, but was read-only. */ WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || txn->mod_count == 0); @@ -1131,7 +1132,8 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) txn_prepared_updates_count = 0; WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); - WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || txn->mod_count == 0); + WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR)); + /* * A transaction should not have updated any of the logged tables, if debug mode logging is not * turned on. @@ -1605,15 +1607,16 @@ __wt_txn_is_blocking(WT_SESSION_IMPL *session) return (false); /* - * Check the oldest transaction ID of either the current transaction ID or the snapshot. Using - * the snapshot potentially means rolling back a read-only transaction, which MongoDB can't - * (yet) handle. For this reason, don't use the snapshot unless there's also a transaction ID - * or we're configured to time out thread operations (a way to confirm our caller is prepared - * for rollback). + * MongoDB can't (yet) handle rolling back read only transactions. For this reason, don't check + * unless there's at least one update or we're configured to time out thread operations (a way + * to confirm our caller is prepared for rollback). */ + if (txn->mod_count == 0 && !__wt_op_timer_fired(session)) + return (false); + + /* Check the oldest transaction ID of either the current transaction ID or the snapshot. */ txn_oldest = txn->id; if (F_ISSET(txn, WT_TXN_HAS_SNAPSHOT) && txn->snap_min != WT_TXN_NONE && - (txn_oldest != WT_TXN_NONE || __wt_op_timer_fired(session)) && (txn_oldest == WT_TXN_NONE || WT_TXNID_LT(txn->snap_min, txn_oldest))) txn_oldest = txn->snap_min; return (txn_oldest == conn->txn_global.oldest_id ? diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index ffa46247106..ea5eeecf75c 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -486,7 +486,7 @@ thread_ts_run(void *arg) printf("SET STABLE: %" PRIx64 " %" PRIu64 "\n", oldest_ts, oldest_ts); } } else - ts_wait: +ts_wait: __wt_sleep(0, 1000); } /* NOTREACHED */ diff --git a/src/third_party/wiredtiger/test/csuite/scope/main.c b/src/third_party/wiredtiger/test/csuite/scope/main.c index 57947fcf166..5a0c50f5a48 100644 --- a/src/third_party/wiredtiger/test/csuite/scope/main.c +++ b/src/third_party/wiredtiger/test/csuite/scope/main.c @@ -50,42 +50,78 @@ handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const ch static WT_EVENT_HANDLER event_handler = {handle_error, NULL, NULL, NULL}; +#define SET_KEY \ + do { \ + if (recno) \ + cursor->set_key(cursor, (uint64_t)1); \ + else { \ + strcpy(keybuf, KEY); \ + cursor->set_key(cursor, keybuf); \ + } \ + } while (0) +#define SET_VALUE \ + do { \ + strcpy(valuebuf, VALUE); \ + if (vstring) \ + cursor->set_value(cursor, valuebuf); \ + else { \ + vu.size = strlen(vu.data = valuebuf); \ + cursor->set_value(cursor, &vu); \ + } \ + } while (0) + static void cursor_scope_ops(WT_SESSION *session, const char *uri) { struct { const char *op; - enum { INSERT, MODIFY, SEARCH, SEARCH_NEAR, REMOVE, REMOVE_POS, RESERVE, UPDATE } func; - const char *config; + enum { + INSERT_GET_KEY, + INSERT_GET_VALUE, + MODIFY, + SEARCH, + SEARCH_NEAR, + REMOVE_GET_KEY, + REMOVE_GET_VALUE, + REMOVE_POS, + RESERVE, + UPDATE + } func; } * op, ops[] = {/* * The ops order is specific: insert has to happen first so * other operations are possible, and remove has to be last. */ { - "insert", INSERT, NULL, + "insert", INSERT_GET_KEY, + }, + { + "insert", INSERT_GET_VALUE, + }, + { + "search", SEARCH, }, { - "search", SEARCH, NULL, + "search", SEARCH_NEAR, }, { - "search", SEARCH_NEAR, NULL, + "reserve", RESERVE, }, { - "reserve", RESERVE, NULL, + "insert", MODIFY, }, { - "insert", MODIFY, NULL, + "update", UPDATE, }, { - "update", UPDATE, NULL, + "remove", REMOVE_GET_KEY, }, { - "remove", REMOVE, NULL, + "remove", REMOVE_GET_VALUE, }, { - "remove", REMOVE_POS, NULL, + "remove", REMOVE_POS, }, - {NULL, INSERT, NULL}}; + {NULL, INSERT_GET_KEY}}; WT_CURSOR *cursor; #define MODIFY_ENTRIES 2 WT_MODIFY entries[MODIFY_ENTRIES]; @@ -94,48 +130,43 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) const char *key, *vs; char keybuf[100], valuebuf[100]; int exact; - bool recno, vstring; - - /* - * Modify and reserve require a transaction, modify requires snapshot isolation. - */ - testutil_check(session->begin_transaction(session, "isolation=snapshot")); + bool recno, rollback, vstring; cursor = NULL; for (op = ops; op->op != NULL; op++) { - key = vs = NULL; - memset(&vu, 0, sizeof(vu)); + /* Open a cursor, track key/value formats. */ + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + recno = strcmp(cursor->key_format, "r") == 0; + vstring = strcmp(cursor->value_format, "S") == 0; + + /* Remove any leftover key/value pair, start fresh. */ + SET_KEY; + testutil_check(cursor->remove(cursor)); + + /* If not an insert operation, make sure there's a key/value pair to operate on. */ + if (op->func != INSERT_GET_KEY && op->func != INSERT_GET_VALUE) { + SET_KEY; + SET_VALUE; + testutil_check(cursor->insert(cursor)); + } + /* Discard that cursor, we'll open one inside the transaction. */ + testutil_check(cursor->close(cursor)); - /* Open a cursor. */ - if (cursor != NULL) - testutil_check(cursor->close(cursor)); - testutil_check(session->open_cursor(session, uri, NULL, op->config, &cursor)); + /* Modify and reserve require a transaction, modify requires snapshot isolation. */ + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + rollback = false; - /* Operations change based on the key/value formats. */ + /* Open a cursor, track key/value formats. */ + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); recno = strcmp(cursor->key_format, "r") == 0; vstring = strcmp(cursor->value_format, "S") == 0; - /* Modify is only possible with "item" values. */ - if (vstring && op->func == MODIFY) - continue; - /* * Set up application buffers so we can detect overwrites or failure to copy application * information into library memory. */ - if (recno) - cursor->set_key(cursor, (uint64_t)1); - else { - strcpy(keybuf, KEY); - cursor->set_key(cursor, keybuf); - } - strcpy(valuebuf, VALUE); - if (vstring) - cursor->set_value(cursor, valuebuf); - else { - vu.size = strlen(vu.data = valuebuf); - cursor->set_value(cursor, &vu); - } + SET_KEY; + SET_VALUE; /* * The application must keep key and value memory valid until the next operation that @@ -144,7 +175,8 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) * Modifying either the key or value buffers is not permitted. */ switch (op->func) { - case INSERT: + case INSERT_GET_KEY: + case INSERT_GET_VALUE: testutil_check(cursor->insert(cursor)); break; case MODIFY: @@ -173,7 +205,8 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) */ testutil_check(cursor->search(cursor)); /* FALLTHROUGH */ - case REMOVE: + case REMOVE_GET_KEY: + case REMOVE_GET_VALUE: testutil_check(cursor->remove(cursor)); break; case RESERVE: @@ -194,20 +227,36 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) /* * Check that get_key/get_value behave as expected after the operation. */ + key = vs = NULL; + keyr = 37; + memset(&vu, 0, sizeof(vu)); switch (op->func) { - case INSERT: - case REMOVE: + case INSERT_GET_KEY: + case REMOVE_GET_KEY: /* * Insert and remove configured with a search key do not position the cursor and have no * key or value. * - * There should be two error messages, ignore them. + * There should be two error messages, ignore them, and errors require rollback. */ - ignore_errors = 2; + ignore_errors = 1; + rollback = true; if (recno) testutil_assert(cursor->get_key(cursor, &keyr) != 0); else testutil_assert(cursor->get_key(cursor, &key) != 0); + testutil_assert(ignore_errors == 0); + break; + case INSERT_GET_VALUE: + case REMOVE_GET_VALUE: + /* + * Insert and remove configured with a search key do not position the cursor and have no + * key or value. + * + * There should be two error messages, ignore them, and errors require rollback. + */ + ignore_errors = 1; + rollback = true; if (vstring) testutil_assert(cursor->get_value(cursor, &vs) != 0); else @@ -218,7 +267,7 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) /* * Remove configured with a cursor position has a key, but no value. * - * There should be one error message, ignore it. + * There should be one error message, ignore it, and errors require rollback. */ if (recno) { testutil_assert(cursor->get_key(cursor, &keyr) == 0); @@ -229,6 +278,7 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) testutil_assert(strcmp(key, KEY) == 0); } ignore_errors = 1; + rollback = true; if (vstring) testutil_assert(cursor->get_value(cursor, &vs) != 0); else @@ -267,25 +317,11 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) break; } - /* - * We have more than one remove operation, add the key back in. - */ - if (op->func == REMOVE || op->func == REMOVE_POS) { - if (recno) - cursor->set_key(cursor, (uint64_t)1); - else { - strcpy(keybuf, KEY); - cursor->set_key(cursor, keybuf); - } - strcpy(valuebuf, VALUE); - if (vstring) - cursor->set_value(cursor, valuebuf); - else { - vu.size = strlen(vu.data = valuebuf); - cursor->set_value(cursor, &vu); - } - testutil_check(cursor->insert(cursor)); - } + if (rollback) + testutil_check(session->rollback_transaction(session, NULL)); + else + testutil_check(session->commit_transaction(session, NULL)); + testutil_check(cursor->close(cursor)); } } diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 971d6b7bcea..d728e9f4e96 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -97,6 +97,36 @@ functions: set -o verbose echo '#undef HAVE_FTRUNCATE' >> wiredtiger_config.h - *make_wiredtiger + "compile wiredtiger address sanitizer": + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + shell: bash + script: | + set -o errexit + set -o verbose + sh reconf + if [ "$OS" != "Windows_NT" ]; then + CC=/opt/mongodbtoolchain/v3/bin/clang CXX=/opt/mongodbtoolchain/v3/bin/clang++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fno-omit-frame-pointer -fsanitize=address" \ + ../configure ${configure_python_setting|} \ + --enable-diagnostic --with-builtins=lz4,snappy,zlib + fi + - *make_wiredtiger + "compile wiredtiger with builtins": + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + shell: bash + script: | + set -o errexit + set -o verbose + sh reconf + if [ "$OS" != "Windows_NT" ]; then + CC=/opt/mongodbtoolchain/v3/bin/gcc CXX=/opt/mongodbtoolchain/v3/bin/g++ PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC" \ + ../configure ${configure_python_setting|} \ + --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib + fi + - *make_wiredtiger "make check directory": command: shell.exec params: @@ -124,7 +154,7 @@ functions: set -o verbose ${test_env_vars|} ${python_binary|python} ../test/suite/run.py ${unit_test_args|-v 2} ${smp_command|} 2>&1 - "test format": + "format test": command: shell.exec params: working_dir: "wiredtiger/build_posix/test/format" @@ -132,32 +162,89 @@ functions: set -o errexit set -o verbose for i in $(seq ${times|1}); do - ./t -1 -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} + ./t -1 -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} 2>&1 + done + "format test script": + command: shell.exec + params: + working_dir: "wiredtiger/test/format" + script: | + set -o errexit + set -o verbose + for i in $(seq ${times|1}); do + ${test_env_vars|} ${format_test_setting|} ./format.sh ${smp_command|} ${format_test_script_args|} 2>&1 done "many dbs test": command: shell.exec - parms: + params: working_dir: "wiredtiger/build_posix/test/manydbs" script: | set -o errexit set -o verbose - ${test_env_vars|} ./t ${many_db_args|} + ${test_env_vars|} ./t ${many_db_args|} 2>&1 "thread test": command: shell.exec - parms: + params: working_dir: "wiredtiger/build_posix/test/thread" script: | set -o errexit set -o verbose - ${test_env_vars|} ./t ${thread_test_args|} + ${test_env_vars|} ./t ${thread_test_args|} 2>&1 "random abort test": command: shell.exec - parms: + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + ${test_env_vars|} ./test_random_abort ${random_abort_args|} 2>&1 + "timestamp abort test": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + ${test_env_vars|} ./test_timestamp_abort ${timestamp_abort_args|} 2>&1 + "truncated log test": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + ${test_env_vars|} ./test_truncated_log ${truncated_log_args|} 2>&1 + "recovery stress test script": + command: shell.exec + params: working_dir: "wiredtiger/build_posix/test/csuite" script: | set -o errexit set -o verbose - ${test_env_vars|} ./test_random_abort ${random_abort_args|} + + for i in $(seq ${times|1}); do + # Run the various combinations of args. Let time and threads be random. + # Run current version with write-no-sync txns. + ${test_env_vars|} ./test_random_abort 2>&1 + ${test_env_vars|} ./test_timestamp_abort 2>&1 + + # Current version with memory-based txns (MongoDB usage). + ${test_env_vars|} ./test_random_abort -m 2>&1 + ${test_env_vars|} ./test_timestamp_abort -m 2>&1 + + # V1 log compatibility mode with write-no-sync txns. + ${test_env_vars|} ./test_random_abort -C 2>&1 + ${test_env_vars|} ./test_timestamp_abort -C 2>&1 + + # V1 log compatibility mode with memory-based txns. + ${test_env_vars|} ./test_random_abort -C -m 2>&1 + ${test_env_vars|} ./test_timestamp_abort -C -m 2>&1 + + ${test_env_vars|} ./test_truncated_log ${truncated_log_args|} 2>&1 + + # Just let the system take a breath + sleep 10s + done "upload artifact": - command: archive.targz_pack params: @@ -191,6 +278,28 @@ functions: set -o verbose ./t ${checkpoint_args} 2>&1 + "checkpoint stress test": + command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/checkpoint" + shell: bash + script: | + set -o errexit + set -o verbose + + export WIREDTIGER_CONFIG='checkpoint_sync=0,transaction_sync=(method=none)' + CMD='./t -h WT_TEST.$i.$t -t r -r 2 -W 3 -n 1000000 -k 1000000 -C "cache_size=100MB"' + + for i in $(seq ${times|1}); do + for t in $(seq ${no_of_procs|1}); do + eval nohup $CMD > nohup.out.$i.$t 2>&1 & + done + + for t in $(seq ${no_of_procs|1}); do + wait -n || exit $? + done + done + pre: - func: "cleanup" post: @@ -632,14 +741,7 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose - - ${test_env_vars|} $(pwd)/test/csuite/test_truncated_log 2>&1 + - func: "truncated log test" - name: csuite-wt1965-col-efficiency-test tags: ["pull_request"] @@ -1337,7 +1439,7 @@ tasks: commands: - func: "fetch artifacts" - func: "compile wiredtiger" - - func: "test format" + - func: "format test" vars: times: 10 config: ../../../test/format/CONFIG.endian @@ -1399,7 +1501,7 @@ tasks: commands: - func: "fetch artifacts" - func: "compile wiredtiger" - - func: "test format" + - func: "format test" vars: times: 10 config: ../../../test/format/CONFIG.endian @@ -1493,7 +1595,7 @@ tasks: commands: - func: "fetch artifacts" - func: "compile wiredtiger" - - func: "test format" + - func: "format test" vars: times: 3 config: ../../../test/format/CONFIG.stress @@ -1507,7 +1609,7 @@ tasks: vars: dependent_task: compile-linux-no-ftruncate - func: "compile wiredtiger no linux ftruncate" - - func: "test format" + - func: "format test" vars: times: 3 @@ -1564,37 +1666,37 @@ tasks: - func: "unit test" vars: unit_test_args: -v 2 --long - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=snappy logging=1 logging_compression=snappy logging_prealloc=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row alter=1 backups=1 compaction=1 data_extend=1 prepare=1 rebalance=1 salvage=1 statistics=1 statistics_server=1 verify=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row direct_io=1 firstfit=1 internal_key_truncation=1 - - func: "test format" + - func: "format test" vars: extra_args: leak_memory=0 mmap=1 file_type=row checkpoints=0 in_memory=1 reverse=1 truncate=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=zlib huffman_key=1 huffman_value=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row isolation=random transaction_timestamps=0 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row data_source=lsm bloom=1 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=var compression=snappy checksum=uncompressed dictionary=1 repeat_data_pct=10 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row compression=lz4 prefix_compression=1 leaf_page_max=9 internal_page_max=9 key_min=256 value_min=256 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=var leaf_page_max=9 internal_page_max=9 value_min=256 - - func: "test format" + - func: "format test" vars: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=fix - command: shell.exec @@ -1623,7 +1725,7 @@ tasks: vars: posix_configure_flags: --enable-python --with-spinlock=gcc --enable-strict - func: "make check all" - - func: "test format" + - func: "format test" vars: times: 3 - func: "unit test" @@ -1635,7 +1737,7 @@ tasks: vars: posix_configure_flags: --enable-python --with-spinlock=pthread_adaptive --enable-strict - func: "make check all" - - func: "test format" + - func: "format test" vars: times: 3 - func: "unit test" @@ -1747,20 +1849,13 @@ tasks: random_abort_args: -t 40 # truncated-log - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix/test/csuite/" - script: | - set -o errexit - set -o verbose - - ./test_truncated_log + - func: "truncated log test" # format test - - func: "test format" + - func: "format test" vars: extra_args: file_type=fix - - func: "test format" + - func: "format test" vars: extra_args: file_type=row @@ -1783,7 +1878,64 @@ tasks: set -o verbose ./time_shift_test.sh /usr/local/lib/faketime/libfaketimeMT.so.1 0-1 2>&1 + + - name: format-stress-sanitizer-test + #set a 25 hours (25*60*60 = 90000 seconds) timeout + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger address sanitizer" + - func: "format test script" + vars: + test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + # run for 24 hours ( 24 * 60 = 1440 minutes), don't stop at failed tests, use default config + format_test_script_args: -t 1440 + + - name: format-stress-sanitizer-smoke-test + #set a 7 hours timeout + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger address sanitizer" + - func: "format test script" + # to emulate the original Jenkins job's test coverage, we are running the smoke test 16 times + # run smoke tests, don't stop at failed tests, use default config + vars: + test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + format_test_script_args: -S + times: 16 + + - name: recovery-stress-test + #set a 25 hours timeout + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=lz4,snappy,zlib + - func: "recovery stress test script" + vars: + # Repeat this script enough times to make this task run for 24 hours + # At the time of writing this script, one call to underlying scripts takes about ~15 mins to finish in worst case. + # We are giving an extra ~20% room for vairance in execution time. + times: 80 + # This is special task until lz4 issues are resolved for zSeries distros + - name: recovery-stress-test-without-lz4 + #set a 25 hours timeout + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=snappy,zlib + - func: "recovery stress test script" + vars: + # Repeat this script enough times to make this task run for 24 hours + # At the time of writing this script, one call to underlying scripts takes about 8 mins to finish in worst case. + # We are giving an extra ~20% room for vairance in execution time. + times: 120 + - name: split-stress-test commands: - func: "get project" @@ -1798,6 +1950,74 @@ tasks: set -o verbose for i in {1..10}; do ${python_binary|python} split_stress.py; done + - name: format-stress-test + # Set 25 hours timeout + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + #run for 24 hours ( 24 * 60 = 1440 minutes), use default config + format_test_script_args: -b "SEGFAULT_SIGNALS=all catchsegv ./t" -t 1440 + + - name: format-stress-smoke-test + # Set 7 hours timeout + exec_timeout_secs: 25200 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + # to emulate the original Jenkins job's test coverage, we are running the smoke test 16 times + # run smoke tests, use default config (-S) + format_test_script_args: -b "SEGFAULT_SIGNALS=all catchsegv ./t" -S + times: 16 + + - name: checkpoint-stress-test + commands: + - command: timeout.update + params: + # set 24 hour timeout for the task and command + exec_timeout_secs: 86400 + timeout_secs: 86400 + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "checkpoint stress test" + vars: + # No of times to run the loop + times: 2 + # No of processes to run in the background + no_of_procs: 10 + + # The task name is ppc-zseries because this task will be used in both buildVariants + - name: format-stress-ppc-zseries-test + # Set 2.5 hours timeout (60 * 60 * 2.5) + exec_timeout_secs: 9000 + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - func: "format test script" + vars: + # Make sure we dump core on failure + format_test_setting: ulimit -c unlimited + #run for 2 hours ( 2 * 60 = 120 minutes), use default config + format_test_script_args: -b "SEGFAULT_SIGNALS=all catchsegv ./t" -t 120 + + - name: format-wtperf-test + commands: + - func: "get project" + - func: "compile wiredtiger with builtins" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/bench/wtperf" + script: | + set -o errexit + set -o verbose + + cp ../../../bench/wtperf/split_heavy.wtperf . + ./wtperf -O ./split_heavy.wtperf -o verbose=2 + buildvariants: - name: ubuntu1804 display_name: Ubuntu 18.04 @@ -1827,7 +2047,13 @@ buildvariants: - name: wtperf-test - name: ftruncate-test - name: long-test + - name: recovery-stress-test + - name: format-stress-sanitizer-test + - name: format-stress-sanitizer-smoke-test - name: split-stress-test + - name: format-stress-test + - name: format-stress-smoke-test + - name: checkpoint-stress-test - name: ubuntu1804-python3 display_name: Ubuntu 18.04 (Python3) @@ -1967,6 +2193,9 @@ buildvariants: tasks: - name: compile - name: unit-test + - name: format-stress-ppc-zseries-test + - name: format-stress-smoke-test + - name: format-wtperf-test - name: ubuntu1804-zseries display_name: Ubuntu 18.04 zSeries @@ -1980,4 +2209,5 @@ buildvariants: tasks: - name: compile - name: unit-test + - name: recovery-stress-test-without-lz4 - name: split-stress-test diff --git a/src/third_party/wiredtiger/test/format/Makefile.am b/src/third_party/wiredtiger/test/format/Makefile.am index 2d9bbf21eb8..bff2986f25e 100644 --- a/src/third_party/wiredtiger/test/format/Makefile.am +++ b/src/third_party/wiredtiger/test/format/Makefile.am @@ -4,7 +4,7 @@ AM_CPPFLAGS +=-I$(top_srcdir)/test/utility noinst_PROGRAMS = t t_SOURCES =\ - backup.c bulk.c compact.c config.c lrt.c ops.c random.c rebalance.c \ + backup.c bulk.c compact.c config.c ops.c random.c rebalance.c \ salvage.c snap.c t.c util.c wts.c t_LDADD = $(top_builddir)/test/utility/libtest_util.la diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 8cec1318efc..9af28f668cd 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -39,7 +39,6 @@ static bool config_fix(void); static void config_in_memory(void); static void config_in_memory_reset(void); static int config_is_perm(const char *); -static void config_lrt(void); static void config_lsm_reset(void); static void config_map_checkpoint(const char *, u_int *); static void config_map_checksum(const char *, u_int *); @@ -176,7 +175,6 @@ config_setup(void) config_compression("compression"); config_compression("logging_compression"); config_encryption(); - config_lrt(); /* Configuration based on the configuration already chosen. */ config_pct(); @@ -238,7 +236,7 @@ config_setup(void) static void config_cache(void) { - uint32_t required; + uint32_t required, workers; /* Page sizes are powers-of-two for bad historic reasons. */ g.intl_page_max = 1U << g.c_intl_page_max; @@ -269,7 +267,8 @@ config_cache(void) * This code is what dramatically increases the cache size when there are lots of threads, it * grows the cache to several megabytes per thread. */ - g.c_cache = WT_MAX(g.c_cache, 2 * g.c_threads * g.c_memory_page_max); + workers = g.c_threads + (g.c_random_cursor ? 1 : 0); + g.c_cache = WT_MAX(g.c_cache, 2 * workers * g.c_memory_page_max); /* * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 @@ -281,7 +280,7 @@ config_cache(void) */ if (DATASOURCE("lsm")) { required = WT_LSM_TREE_MINIMUM_SIZE( - g.c_chunk_size * WT_MEGABYTE, g.c_threads * g.c_merge_max, g.c_threads * g.leaf_page_max); + g.c_chunk_size * WT_MEGABYTE, workers * g.c_merge_max, workers * g.leaf_page_max); required = (required + (WT_MEGABYTE - 1)) / WT_MEGABYTE; if (g.c_cache < required) g.c_cache = required; @@ -451,12 +450,7 @@ config_encryption(void) static bool config_fix(void) { - /* - * Fixed-length column stores don't support the lookaside table (so, no long running - * transactions), or modify operations. - */ - if (config_is_perm("long_running_txn")) - return (false); + /* Fixed-length column stores don't support the lookaside table, so no modify operations. */ if (config_is_perm("modify_pct")) return (false); return (true); @@ -573,25 +567,6 @@ config_lsm_reset(void) } /* - * config_lrt -- - * Long-running transaction configuration. - */ -static void -config_lrt(void) -{ - /* - * WiredTiger doesn't support a lookaside file for fixed-length column stores. - */ - if (g.type == FIX && g.c_long_running_txn) { - if (config_is_perm("long_running_txn")) - testutil_die(EINVAL, - "long_running_txn not supported with fixed-length " - "column store"); - config_single("long_running_txn=off", false); - } -} - -/* * config_pct -- * Configure operation percentages. */ diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 492d5124a1c..4c9587d5146 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -190,9 +190,6 @@ static CONFIG c[] = {{"abort", "if timed run should drop core", /* 0% */ {"logging_prealloc", "if log file pre-allocation configured", /* 50% */ C_BOOL, 50, 0, 0, &g.c_logging_prealloc, NULL}, - {"long_running_txn", "if a long-running transaction configured", /* 0% */ - C_BOOL, 0, 0, 0, &g.c_long_running_txn, NULL}, - {"lsm_worker_threads", "the number of LSM worker threads", 0x0, 3, 4, 20, &g.c_lsm_worker_threads, NULL}, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 38755e47564..a1fa5d425b9 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -169,7 +169,6 @@ typedef struct { char *c_logging_compression; uint32_t c_logging_file_max; uint32_t c_logging_prealloc; - uint32_t c_long_running_txn; uint32_t c_lsm_worker_threads; uint32_t c_memory_page_max; uint32_t c_merge_max; @@ -347,7 +346,6 @@ void key_gen_init(WT_ITEM *); void key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t); void key_gen_teardown(WT_ITEM *); void key_init(void); -WT_THREAD_RET lrt(void *); WT_THREAD_RET random_kv(void *); void path_setup(const char *); int read_row_worker(WT_CURSOR *, uint64_t, WT_ITEM *, WT_ITEM *, bool); diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index 3a140bcfc05..37607d6b8df 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -48,9 +48,6 @@ smoke_list=( "$smoke_base_1 file_type=row huffman_key=1 huffman_value=1" "$smoke_base_1 file_type=var huffman_key=1 huffman_value=1" - # Abort/recovery test. - "$smoke_base_1 file_type=row abort=1" - # LSM "$smoke_base_1 file_type=row data_source=lsm" diff --git a/src/third_party/wiredtiger/test/format/lrt.c b/src/third_party/wiredtiger/test/format/lrt.c deleted file mode 100644 index 472a8a0d877..00000000000 --- a/src/third_party/wiredtiger/test/format/lrt.c +++ /dev/null @@ -1,175 +0,0 @@ -/*- - * Public Domain 2014-2019 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "format.h" - -/* - * lrt -- - * Start a long-running transaction. - */ -WT_THREAD_RET -lrt(void *arg) -{ - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_ITEM key, value; - WT_SESSION *session; - size_t buf_len, buf_size; - uint64_t keyno, saved_keyno; - uint8_t bitfield; - u_int period; - int pinned, ret; - void *buf; - - (void)(arg); /* Unused parameter */ - - saved_keyno = 0; /* [-Werror=maybe-uninitialized] */ - - key_gen_init(&key); - val_gen_init(&value); - - buf = NULL; - buf_len = buf_size = 0; - - /* Open a session and cursor. */ - conn = g.wts_conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); - - for (pinned = 0;;) { - if (pinned) { - /* Re-read the record at the end of the table. */ - while ((ret = read_row_worker(cursor, saved_keyno, &key, &value, false)) == WT_ROLLBACK) - ; - if (ret != 0) - testutil_die(ret, "read_row_worker %" PRIu64, saved_keyno); - - /* Compare the previous value with the current one. */ - if (g.type == FIX) { - ret = cursor->get_value(cursor, &bitfield); - value.data = &bitfield; - value.size = 1; - } else - ret = cursor->get_value(cursor, &value); - if (ret != 0) - testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); - - if (buf_size != value.size || memcmp(buf, value.data, value.size) != 0) - testutil_die(0, "mismatched start/stop values"); - - /* End the transaction. */ - testutil_check(session->commit_transaction(session, NULL)); - - /* Reset the cursor, releasing our pin. */ - testutil_check(cursor->reset(cursor)); - pinned = 0; - } else { - /* - * Test named snapshots: create a snapshot, wait to give the transaction state time to - * move forward, then start a transaction with the named snapshot, drop it, then commit - * the transaction. This exercises most of the named snapshot logic under load. - */ - testutil_check(session->snapshot(session, "name=test")); - __wt_sleep(1, 0); - wiredtiger_begin_transaction(session, "snapshot=test"); - testutil_check(session->snapshot(session, "drop=(all)")); - testutil_check(session->commit_transaction(session, NULL)); - - /* - * Begin transaction: without an explicit transaction, the snapshot is only kept around - * while a cursor is positioned. As soon as the cursor loses its position a new snapshot - * will be allocated. - */ - while ( - (ret = session->begin_transaction(session, "snapshot=snapshot")) == WT_CACHE_FULL) - ; - testutil_check(ret); - - /* Read a record at the end of the table. */ - do { - saved_keyno = mmrand(NULL, (u_int)(g.key_cnt - g.key_cnt / 10), (u_int)g.key_cnt); - while ( - (ret = read_row_worker(cursor, saved_keyno, &key, &value, false)) == WT_ROLLBACK) - ; - } while (ret == WT_NOTFOUND); - if (ret != 0) - testutil_die(ret, "read_row_worker %" PRIu64, saved_keyno); - - /* Copy the cursor's value. */ - if (g.type == FIX) { - ret = cursor->get_value(cursor, &bitfield); - value.data = &bitfield; - value.size = 1; - } else - ret = cursor->get_value(cursor, &value); - if (ret != 0) - testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); - if (buf_len < value.size) - buf = drealloc(buf, buf_len = value.size); - memcpy(buf, value.data, buf_size = value.size); - - /* - * Move the cursor to an early record in the table, hopefully allowing the page with the - * record just retrieved to be evicted from memory. - */ - do { - keyno = mmrand(NULL, 1, (u_int)g.key_cnt / 5); - while ((ret = read_row_worker(cursor, keyno, &key, &value, false)) == WT_ROLLBACK) - ; - } while (ret == WT_NOTFOUND); - if (ret != 0) - testutil_die(ret, "read_row_worker %" PRIu64, keyno); - - pinned = 1; - } - - /* Sleep for some number of seconds. */ - period = mmrand(NULL, 1, 10); - - /* Sleep for short periods so we don't make the run wait. */ - while (period > 0 && !g.workers_finished) { - --period; - __wt_sleep(1, 0); - } - if (g.workers_finished) - break; - } - - testutil_check(session->close(session, NULL)); - - key_gen_teardown(&key); - val_gen_teardown(&value); - free(buf); - - return (WT_THREAD_RET_VALUE); -} diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 2b4ae05cc5f..669cb7a484e 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -126,7 +126,7 @@ wts_ops(bool lastrun) TINFO *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, lrt_tid, random_tid; + wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, random_tid; wt_thread_t timestamp_tid; int64_t fourths, quit_fourths, thread_ops; uint32_t i; @@ -139,7 +139,6 @@ wts_ops(bool lastrun) memset(&backup_tid, 0, sizeof(backup_tid)); memset(&checkpoint_tid, 0, sizeof(checkpoint_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); - memset(&lrt_tid, 0, sizeof(lrt_tid)); memset(&random_tid, 0, sizeof(random_tid)); memset(×tamp_tid, 0, sizeof(timestamp_tid)); @@ -209,7 +208,7 @@ wts_ops(bool lastrun) } /* - * If a multi-threaded run, start optional backup, compaction and long-running reader threads. + * If a multi-threaded run, start optional special-purpose threads. */ if (g.c_alter) testutil_check(__wt_thread_create(NULL, &alter_tid, alter, NULL)); @@ -219,8 +218,6 @@ wts_ops(bool lastrun) testutil_check(__wt_thread_create(NULL, &checkpoint_tid, checkpoint, NULL)); if (g.c_compact) testutil_check(__wt_thread_create(NULL, &compact_tid, compact, NULL)); - if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL)); if (g.c_random_cursor) testutil_check(__wt_thread_create(NULL, &random_tid, random_kv, NULL)); if (g.c_txn_timestamps) @@ -293,7 +290,7 @@ wts_ops(bool lastrun) } } - /* Wait for the other threads. */ + /* Wait for the special-purpose threads. */ g.workers_finished = true; if (g.c_alter) testutil_check(__wt_thread_join(NULL, &alter_tid)); @@ -303,8 +300,6 @@ wts_ops(bool lastrun) testutil_check(__wt_thread_join(NULL, &checkpoint_tid)); if (g.c_compact) testutil_check(__wt_thread_join(NULL, &compact_tid)); - if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(__wt_thread_join(NULL, &lrt_tid)); if (g.c_random_cursor) testutil_check(__wt_thread_join(NULL, &random_tid)); if (g.c_txn_timestamps) @@ -842,7 +837,7 @@ ops(void *arg) READ_OP_FAILED(true); break; case REMOVE: - remove_instead_of_truncate: +remove_instead_of_truncate: switch (g.type) { case ROW: ret = row_remove(tinfo, cursor, positioned); @@ -930,7 +925,7 @@ ops(void *arg) WRITE_OP_FAILED(false); break; case UPDATE: - update_instead_of_chosen_op: +update_instead_of_chosen_op: ++tinfo->update; switch (g.type) { case ROW: @@ -1250,7 +1245,7 @@ nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) } else if (tinfo->keyno > keyno || (!record_gaps && keyno != tinfo->keyno + 1)) goto order_error_col; if (0) { - order_error_col: +order_error_col: testutil_die( 0, "%s returned %" PRIu64 " then %" PRIu64, which, tinfo->keyno, keyno); } @@ -1284,7 +1279,7 @@ nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) goto order_error_row; } if (0) { - order_error_row: +order_error_row: testutil_die(0, "%s returned {%.*s} then {%.*s}", which, (int)tinfo->key->size, (char *)tinfo->key->data, (int)key.size, (char *)key.data); } diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode06.py b/src/third_party/wiredtiger/test/suite/test_debug_mode06.py new file mode 100644 index 00000000000..355102fcebc --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode06.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest, wiredtiger + +# test_debug_mode06.py +# Test the debug mode settings. Test slow_checkpoint use (from WT-4981). +# Note: testing timing will make results unreliable so we won't do that. +class test_debug_mode06(wttest.WiredTigerTestCase): + conn_config = 'log=(enabled=true),debug_mode=(slow_checkpoint=true),statistics=(all)' + uri = 'file:test_debug_mode06' + + # Insert some data to ensure setting/unsetting the flag does not + # break existing functionality + def insert_data(self, assert_time=0): + self.session.create(self.uri, 'key_format=s,value_format=s') + self.cursor = self.session.open_cursor(self.uri, None) + self.cursor['key'] = 'value' + self.cursor.close() + self.session.checkpoint() + + # Validate checkpoint time if asked for. + if (assert_time > 0): + stat_cur = self.session.open_cursor('statistics:', None, None) + checkpoint_time = int(stat_cur[wiredtiger.stat.conn.txn_checkpoint_time_recent][2]) + self.assertTrue(checkpoint_time >= assert_time) + stat_cur.close() + + # Make flag works when set. + def test_slow_checkpoints(self): + # Ensure the checkpoint takes at least 10ms (the delay we have set). + self.insert_data(10) + + # Make sure the flag can be 'turned off' as well. + def test_slow_checkpoints_off(self): + conn_reconfig = 'debug_mode=(slow_checkpoint=false)' + self.conn.reconfigure(conn_reconfig) + self.insert_data() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode07.py b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py new file mode 100644 index 00000000000..f4a34307579 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode07.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest, wiredtiger + +# test_debug_mode07.py +# Test the debug mode settings. Test realloc_exact use (from WT-4919). +class test_debug_mode07(wttest.WiredTigerTestCase): + conn_config = 'log=(enabled=true),debug_mode=(realloc_exact=true),statistics=(all)' + uri = 'file:test_debug_mode07' + + # Insert some data to ensure setting/unsetting the flag does not + # break existing functionality. Also call checkpoint because it + # causes the realloc function to be called numerous times. + def insert_data(self): + self.session.create(self.uri, 'key_format=s,value_format=s') + self.cursor = self.session.open_cursor(self.uri, None) + self.cursor['key'] = 'value' + self.cursor.close() + self.session.checkpoint() + + # Make flag works when set. + def test_realloc_exact(self): + self.insert_data() + + # Make sure the flag can be 'turned off' as well. + def test_realloc_exact_off(self): + conn_reconfig = 'debug_mode=(realloc_exact=false)' + self.conn.reconfigure(conn_reconfig) + self.insert_data() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py index 1734efecd0e..e1db60bbd44 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py @@ -62,7 +62,6 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(2)), '/older than the first commit timestamp/') - c[2] = 2 self.session.rollback_transaction() self.session.begin_transaction() @@ -87,7 +86,6 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(2)), '/less than the oldest timestamp/') - c[2] = 2 self.session.rollback_transaction() self.session.begin_transaction() @@ -137,7 +135,6 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(5)), '/less than the stable timestamp/') - c[5] = 5 self.session.rollback_transaction() self.session.begin_transaction() |