diff options
author | Luke Chen <luke.chen@mongodb.com> | 2020-01-15 05:39:47 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2020-01-15 05:39:47 +0000 |
commit | 44d463b0c7f6596bdd86b2d015db9f79bc434f4b (patch) | |
tree | f038f8f1daf7ce592ec9ad0621b18c800ee26771 /src/third_party/wiredtiger | |
parent | f22f71a0d3b10a8bfcf69d078621b240a67cf515 (diff) | |
download | mongo-44d463b0c7f6596bdd86b2d015db9f79bc434f4b.tar.gz |
Import wiredtiger: c6962452f2c3469c039edfcafb7702450bf14c3b from branch mongodb-4.4
ref: 5bd9fdfc0b..c6962452f2
for: 4.3.3
WT-4886 Avoid depending on particular error message text for test_bug018
WT-5003 Migrate Jenkins “wiredtiger-test-race-condition-stress-sanitizer” job to Evergreen
WT-5024 Migrate Jenkins “wiredtiger-compatibility” job to Evergreen
WT-5081 Add support for ASan builds in the Python test suite
WT-5119 Birthmark records can be read as normal updates if reads race with checkpoints
WT-5165 Add cursor copy debug mode
WT-5199 Adding correctness tasks into rhel80 Evergreen build variant
WT-5261 coverage-report task timed out in Evergreen
WT-5334 Add testing coverage for static wt build
WT-5379 Missing varargs cleanup
WT-5410 WiredTiger formatted timestamp string buffer size is too small
Diffstat (limited to 'src/third_party/wiredtiger')
25 files changed, 685 insertions, 146 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 3a46d6e7ff0..cf942be2f84 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -495,6 +495,13 @@ connection_runtime_config = [ adjust log archiving to retain the log records of this number of checkpoints. Zero or one means perform normal archiving.''', min='0', max='1024'), + Config('cursor_copy', 'false', r''' + if true, use the system allocator to make a copy of any data + returned by a cursor operation and return the copy instead. + The copy is freed on the next cursor operation. This allows + memory sanitizers to detect inappropriate references to memory + owned by cursors.''', + type='boolean'), Config('eviction', 'false', r''' if true, modify internal algorithms to change skew to force lookaside eviction to happen more aggressively. This includes but diff --git a/src/third_party/wiredtiger/dist/s_function b/src/third_party/wiredtiger/dist/s_function index c16dcaf67e8..81d5795afd4 100755 --- a/src/third_party/wiredtiger/dist/s_function +++ b/src/third_party/wiredtiger/dist/s_function @@ -53,22 +53,6 @@ for f in `find bench examples ext src test -name '*.[ci]'`; do } done -# Jumps inside va_start/va_end pairs. -for f in `find bench examples ext src test -name '*.[ci]'`; do - file_parse $f | - egrep 'va_start.*(WT_RET|goto).*va_end' | - sed 's/:.*//' > $t - file_parse $f | - egrep -v 'va_start.*WT_ERR.*[^a-z_]err:.*va_end' | - egrep 'va_start.*WT_ERR.*va_end' | - sed 's/:.*//' >> $t - - test -s $t && { - echo "$f: va_end of va_start/va_end pair can be skipped" - sed 's/^/function @ line:/' < $t - } -done - # Early exits from critical loops. for f in `find bench examples ext src test -name '*.[ci]'`; do sed -n -e '/API_CALL.*;$/,/API_END.*;/{=;p;}' \ diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index b2f18ef2ef2..6abe03cba6b 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,6 +1,6 @@ { - "commit": "5bd9fdfc0bb7f8fcf791e234b6e436c79dfe1fee", - "github": "wiredtiger/wiredtiger.git", - "vendor": "wiredtiger", - "branch": "mongodb-4.4" + "vendor": "wiredtiger", + "github": "wiredtiger/wiredtiger.git", + "branch": "mongodb-4.4", + "commit": "c6962452f2c3469c039edfcafb7702450bf14c3b" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 12f510728c7..629c5316e53 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -54,7 +54,8 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_compatibility_sub static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = { {"checkpoint_retention", "int", NULL, "min=0,max=1024", NULL, 0}, - {"eviction", "boolean", NULL, NULL, NULL, 0}, {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, + {"cursor_copy", "boolean", NULL, NULL, NULL, 0}, {"eviction", "boolean", NULL, NULL, NULL, 0}, + {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, {"rollback_error", "int", NULL, "min=0,max=10M", NULL, 0}, {"slow_checkpoint", "boolean", NULL, NULL, NULL, 0}, {"table_logging", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -105,7 +106,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"compatibility", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, {"error_prefix", "string", NULL, NULL, NULL, 0}, {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2}, {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0}, @@ -541,7 +542,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -608,7 +609,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -675,7 +676,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -740,7 +741,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 6}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -815,13 +816,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "async=(enabled=false,ops_max=1024,threads=2),cache_max_wait_ms=0" ",cache_overflow=(file_max=0),cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),compatibility=(release=)," - "debug_mode=(checkpoint_retention=0,eviction=false," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),error_prefix=,eviction=(threads_max=8," - "threads_min=1),eviction_checkpoint_target=1," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95," - "file_manager=(close_handle_minimum=250,close_idle_time=30," + "debug_mode=(checkpoint_retention=0,cursor_copy=false," + "eviction=false,realloc_exact=false,rollback_error=0," + "slow_checkpoint=false,table_logging=false),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),io_capacity=(total=0),log=(archive=true," "os_cache_dirty_pct=0,prealloc=true,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4)," @@ -989,11 +990,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," - "debug_mode=(checkpoint_retention=0,eviction=false," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "debug_mode=(checkpoint_retention=0,cursor_copy=false," + "eviction=false,realloc_exact=false,rollback_error=0," + "slow_checkpoint=false,table_logging=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1019,11 +1021,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," - "debug_mode=(checkpoint_retention=0,eviction=false," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "debug_mode=(checkpoint_retention=0,cursor_copy=false," + "eviction=false,realloc_exact=false,rollback_error=0," + "slow_checkpoint=false,table_logging=false),direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",exclusive=false,extensions=,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1048,11 +1051,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8" ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," - "require_min=),debug_mode=(checkpoint_retention=0,eviction=false," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "require_min=),debug_mode=(checkpoint_retention=0," + "cursor_copy=false,eviction=false,realloc_exact=false," + "rollback_error=0,slow_checkpoint=false,table_logging=false)," + "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," @@ -1075,11 +1079,12 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "cache_max_wait_ms=0,cache_overflow=(file_max=0),cache_overhead=8" ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," - "require_min=),debug_mode=(checkpoint_retention=0,eviction=false," - "realloc_exact=false,rollback_error=0,slow_checkpoint=false," - "table_logging=false),direct_io=,encryption=(keyid=,name=," - "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" - ",eviction_checkpoint_target=1,eviction_dirty_target=5," + "require_min=),debug_mode=(checkpoint_retention=0," + "cursor_copy=false,eviction=false,realloc_exact=false," + "rollback_error=0,slow_checkpoint=false,table_logging=false)," + "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 7013ca58238..ace1505e6dd 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1790,6 +1790,12 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) else WT_RET(__wt_calloc_def(session, conn->debug_ckpt_cnt, &conn->debug_ckpt)); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.cursor_copy", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_DEBUG_CURSOR_COPY); + else + F_CLR(conn, WT_CONN_DEBUG_CURSOR_COPY); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.eviction", &cval)); if (cval.val) F_SET(cache, WT_CACHE_EVICT_DEBUG_MODE); diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index 9006690402b..cb6fa445ae3 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -60,7 +60,6 @@ __curdump_get_key(WT_CURSOR *cursor, ...) cdump = (WT_CURSOR_DUMP *)cursor; child = cdump->child; - va_start(ap, cursor); CURSOR_API_CALL(cursor, session, get_key, NULL); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) { @@ -80,7 +79,9 @@ __curdump_get_key(WT_CURSOR *cursor, ...) else fmt = cursor->key_format; } + va_start(ap, cursor); ret = __wt_json_alloc_unpack(session, buffer, size, fmt, json, true, ap); + va_end(ap); } else { if (WT_CURSOR_RECNO(cursor) && !F_ISSET(cursor, WT_CURSTD_RAW)) { WT_ERR(child->get_key(child, &recno)); @@ -93,16 +94,17 @@ __curdump_get_key(WT_CURSOR *cursor, ...) __raw_to_dump(session, &item, &cursor->key, F_ISSET(cursor, WT_CURSTD_DUMP_HEX))); } + va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSTD_RAW)) { itemp = va_arg(ap, WT_ITEM *); itemp->data = cursor->key.data; itemp->size = cursor->key.size; } else *va_arg(ap, const char **) = cursor->key.data; + va_end(ap); } err: - va_end(ap); API_END_RET(session, ret); } @@ -221,7 +223,6 @@ __curdump_get_value(WT_CURSOR *cursor, ...) cdump = (WT_CURSOR_DUMP *)cursor; child = cdump->child; - va_start(ap, cursor); CURSOR_API_CALL(cursor, session, get_value, NULL); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) { @@ -229,22 +230,25 @@ __curdump_get_value(WT_CURSOR *cursor, ...) WT_ASSERT(session, json != NULL); WT_ERR(__wt_cursor_get_raw_value(child, &item)); fmt = F_ISSET(cursor, WT_CURSTD_RAW) ? "u" : cursor->value_format; + va_start(ap, cursor); ret = __wt_json_alloc_unpack(session, item.data, item.size, fmt, json, false, ap); + va_end(ap); } else { WT_ERR(child->get_value(child, &item)); WT_ERR(__raw_to_dump(session, &item, &cursor->value, F_ISSET(cursor, WT_CURSTD_DUMP_HEX))); + va_start(ap, cursor); if (F_ISSET(cursor, WT_CURSTD_RAW)) { itemp = va_arg(ap, WT_ITEM *); itemp->data = cursor->value.data; itemp->size = cursor->value.size; } else *va_arg(ap, const char **) = cursor->value.data; + va_end(ap); } err: - va_end(ap); API_END_RET(session, ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 30fe571bdbc..32adec88787 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -88,6 +88,7 @@ __curfile_next(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, next, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__wt_btcur_next(cbt, false)); @@ -114,6 +115,7 @@ __wt_curfile_next_random(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, next, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__wt_btcur_next_random(cbt)); @@ -139,6 +141,7 @@ __curfile_prev(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, prev, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__wt_btcur_prev(cbt, false)); @@ -164,6 +167,7 @@ __curfile_reset(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, reset, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); ret = __wt_btcur_reset(cbt); @@ -189,6 +193,7 @@ __curfile_search(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, search, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__cursor_checkkey(cursor)); time_start = __wt_clock(session); @@ -219,6 +224,7 @@ __curfile_search_near(WT_CURSOR *cursor, int *exact) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, search_near, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__cursor_checkkey(cursor)); time_start = __wt_clock(session); @@ -249,6 +255,7 @@ __curfile_insert(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL_BTREE(cursor, session, insert, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); if (!F_ISSET(cursor, WT_CURSTD_APPEND)) WT_ERR(__cursor_checkkey(cursor)); @@ -289,6 +296,7 @@ __wt_curfile_insert_check(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; tret = 0; CURSOR_UPDATE_API_CALL_BTREE(cursor, session, update, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__cursor_checkkey(cursor)); tret = __wt_btcur_insert_check(cbt); @@ -315,6 +323,7 @@ __curfile_modify(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL_BTREE(cursor, session, modify, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__cursor_checkkey(cursor)); /* Check for a rational modify vector count. */ @@ -349,6 +358,7 @@ __curfile_update(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL_BTREE(cursor, session, update, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__cursor_checkkey(cursor)); WT_ERR(__cursor_checkvalue(cursor)); @@ -391,6 +401,7 @@ __curfile_remove(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_REMOVE_API_CALL(cursor, session, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__cursor_checkkey(cursor)); time_start = __wt_clock(session); @@ -427,6 +438,7 @@ __curfile_reserve(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL_BTREE(cursor, session, reserve, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); WT_ERR(__cursor_checkkey(cursor)); WT_ERR(__wt_txn_context_check(session, true)); @@ -470,6 +482,7 @@ __curfile_close(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, close, cbt->btree); + WT_ERR(__cursor_copy_release(cursor)); err: /* Only try to cache the cursor if there's no error. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index 8de5878653e..8f4cf05937d 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -19,12 +19,13 @@ __curindex_get_value(WT_CURSOR *cursor, ...) WT_SESSION_IMPL *session; va_list ap; - va_start(ap, cursor); JOINABLE_CURSOR_API_CALL(cursor, session, get_value, NULL); - WT_ERR(__wt_curindex_get_valuev(cursor, ap)); -err: + va_start(ap, cursor); + ret = __wt_curindex_get_valuev(cursor, ap); va_end(ap); + +err: API_END_RET(session, ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 5b2dc711a7e..26bf014d51c 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -678,15 +678,15 @@ __curjoin_get_key(WT_CURSOR *cursor, ...) cjoin = (WT_CURSOR_JOIN *)cursor; - va_start(ap, cursor); JOINABLE_CURSOR_API_CALL(cursor, session, get_key, NULL); if (!F_ISSET(cjoin, WT_CURJOIN_INITIALIZED) || !cjoin->iter->positioned) WT_ERR_MSG(session, EINVAL, "join cursor must be advanced with next()"); - WT_ERR(__wt_cursor_get_keyv(cursor, cursor->flags, ap)); + va_start(ap, cursor); + ret = __wt_cursor_get_keyv(cursor, cursor->flags, ap); + va_end(ap); err: - va_end(ap); API_END_RET(session, ret); } @@ -704,16 +704,16 @@ __curjoin_get_value(WT_CURSOR *cursor, ...) cjoin = (WT_CURSOR_JOIN *)cursor; - va_start(ap, cursor); JOINABLE_CURSOR_API_CALL(cursor, session, get_value, NULL); if (!F_ISSET(cjoin, WT_CURJOIN_INITIALIZED) || !cjoin->iter->positioned) WT_ERR_MSG(session, EINVAL, "join cursor must be advanced with next()"); - WT_ERR(__wt_curtable_get_valuev(cjoin->main, ap)); + va_start(ap, cursor); + ret = __wt_curtable_get_valuev(cjoin->main, ap); + va_end(ap); err: - va_end(ap); API_END_RET(session, ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index fedf1b07640..01858ce6f97 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -48,7 +48,6 @@ __curstat_get_key(WT_CURSOR *cursor, ...) va_list ap; cst = (WT_CURSOR_STAT *)cursor; - va_start(ap, cursor); CURSOR_API_CALL(cursor, session, get_key, NULL); WT_ERR(__cursor_needkey(cursor)); @@ -58,14 +57,18 @@ __curstat_get_key(WT_CURSOR *cursor, ...) WT_ERR(__wt_buf_initsize(session, &cursor->key, size)); WT_ERR(__wt_struct_pack(session, cursor->key.mem, size, cursor->key_format, cst->key)); + va_start(ap, cursor); item = va_arg(ap, WT_ITEM *); item->data = cursor->key.data; item->size = cursor->key.size; - } else + va_end(ap); + } else { + va_start(ap, cursor); *va_arg(ap, int *) = cst->key; + va_end(ap); + } err: - va_end(ap); API_END_RET(session, ret); } @@ -86,7 +89,6 @@ __curstat_get_value(WT_CURSOR *cursor, ...) va_list ap; cst = (WT_CURSOR_STAT *)cursor; - va_start(ap, cursor); CURSOR_API_CALL(cursor, session, get_value, NULL); WT_ERR(__cursor_needvalue(cursor)); @@ -98,24 +100,27 @@ __curstat_get_value(WT_CURSOR *cursor, ...) WT_ERR(__wt_struct_pack( session, cursor->value.mem, size, cursor->value_format, desc, cst->pv.data, cst->v)); + va_start(ap, cursor); item = va_arg(ap, WT_ITEM *); item->data = cursor->value.data; item->size = cursor->value.size; + va_end(ap); } else { /* * Don't drop core if the statistics value isn't requested; NULL pointer support isn't * documented, but it's a cheap test. */ + va_start(ap, cursor); if ((p = va_arg(ap, const char **)) != NULL) *p = desc; if ((p = va_arg(ap, const char **)) != NULL) *p = cst->pv.data; if ((v = va_arg(ap, uint64_t *)) != NULL) *v = cst->v; + va_end(ap); } err: - va_end(ap); API_END_RET(session, ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 7ff9273bf6d..84224560b02 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -228,6 +228,34 @@ __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_ATTRIBUTE((cold) } /* + * __wt_cursor_copy_release_item -- + * Release memory used by the key or value item in cursor copy debug mode. + */ +int +__wt_cursor_copy_release_item(WT_CURSOR *cursor, WT_ITEM *item) WT_GCC_FUNC_ATTRIBUTE((cold)) +{ + WT_DECL_ITEM(tmp); + WT_DECL_RET; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)cursor->session; + + /* + * Whether or not we own the memory for the item, make a copy of the data and use that instead. + * That allows us to overwrite and free memory owned by the item, potentially uncovering + * programming errors related to retaining pointers to key/value memory beyond API boundaries. + */ + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_buf_set(session, tmp, item->data, item->size)); + __wt_explicit_overwrite(item->mem, item->memsize); + __wt_buf_free(session, item); + WT_ERR(__wt_buf_set(session, item, tmp->data, tmp->size)); +err: + __wt_scr_free(session, &tmp); + return (ret); +} + +/* * __wt_cursor_get_key -- * WT_CURSOR->get_key default implementation. */ @@ -346,6 +374,10 @@ __wt_cursor_get_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) if (!F_ISSET(cursor, WT_CURSTD_KEY_SET)) WT_ERR(__wt_cursor_kv_not_set(cursor, true)); + /* Force an allocated copy when using cursor copy debug. */ + if (F_ISSET(S2C(session), WT_CONN_DEBUG_CURSOR_COPY)) + WT_ERR(__wt_buf_grow(session, &cursor->key, cursor->key.size)); + if (WT_CURSOR_RECNO(cursor)) { if (LF_ISSET(WT_CURSTD_RAW)) { key = va_arg(ap, WT_ITEM *); @@ -391,6 +423,7 @@ __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) tmp.mem = NULL; CURSOR_API_CALL(cursor, session, set_key, NULL); + WT_ERR(__cursor_copy_release(cursor)); if (F_ISSET(cursor, WT_CURSTD_KEY_SET) && WT_DATA_IN_ITEM(buf)) { tmp = *buf; buf->mem = NULL; @@ -447,7 +480,7 @@ err: * memory in the meantime, free it. */ if (tmp.mem != NULL) { - if (buf->mem == NULL) { + if (buf->mem == NULL && !F_ISSET(S2C(session), WT_CONN_DEBUG_CURSOR_COPY)) { buf->mem = tmp.mem; buf->memsize = tmp.memsize; } else @@ -489,6 +522,10 @@ __wt_cursor_get_valuev(WT_CURSOR *cursor, va_list ap) if (!F_ISSET(cursor, WT_CURSTD_VALUE_EXT | WT_CURSTD_VALUE_INT)) WT_ERR(__wt_cursor_kv_not_set(cursor, false)); + /* Force an allocated copy when using cursor copy debug. */ + if (F_ISSET(S2C(session), WT_CONN_DEBUG_CURSOR_COPY)) + WT_ERR(__wt_buf_grow(session, &cursor->value, cursor->value.size)); + /* Fast path some common cases. */ fmt = cursor->value_format; if (F_ISSET(cursor, WT_CURSOR_RAW_OK) || WT_STREQ(fmt, "u")) { @@ -538,6 +575,7 @@ __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap) tmp.mem = NULL; CURSOR_API_CALL(cursor, session, set_value, NULL); + WT_ERR(__cursor_copy_release(cursor)); if (F_ISSET(cursor, WT_CURSTD_VALUE_SET) && WT_DATA_IN_ITEM(buf)) { tmp = *buf; buf->mem = NULL; @@ -581,7 +619,7 @@ err: * memory in the meantime, free it. */ if (tmp.mem != NULL) { - if (buf->mem == NULL) { + if (buf->mem == NULL && !F_ISSET(S2C(session), WT_CONN_DEBUG_CURSOR_COPY)) { buf->mem = tmp.mem; buf->memsize = tmp.memsize; } else diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index f77712972b6..9498eb5d6c6 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -519,29 +519,30 @@ struct __wt_connection_impl { #define WT_CONN_CLOSING_TIMESTAMP 0x00000020u #define WT_CONN_COMPATIBILITY 0x00000040u #define WT_CONN_DATA_CORRUPTION 0x00000080u -#define WT_CONN_DEBUG_REALLOC_EXACT 0x00000100u -#define WT_CONN_DEBUG_SLOW_CKPT 0x00000200u -#define WT_CONN_EVICTION_NO_LOOKASIDE 0x00000400u -#define WT_CONN_EVICTION_RUN 0x00000800u -#define WT_CONN_INCR_BACKUP 0x00001000u -#define WT_CONN_IN_MEMORY 0x00002000u -#define WT_CONN_LEAK_MEMORY 0x00004000u -#define WT_CONN_LOOKASIDE_OPEN 0x00008000u -#define WT_CONN_LSM_MERGE 0x00010000u -#define WT_CONN_OPTRACK 0x00020000u -#define WT_CONN_PANIC 0x00040000u -#define WT_CONN_READONLY 0x00080000u -#define WT_CONN_RECONFIGURING 0x00100000u -#define WT_CONN_RECOVERING 0x00200000u -#define WT_CONN_SALVAGE 0x00400000u -#define WT_CONN_SERVER_ASYNC 0x00800000u -#define WT_CONN_SERVER_CAPACITY 0x01000000u -#define WT_CONN_SERVER_CHECKPOINT 0x02000000u -#define WT_CONN_SERVER_LOG 0x04000000u -#define WT_CONN_SERVER_LSM 0x08000000u -#define WT_CONN_SERVER_STATISTICS 0x10000000u -#define WT_CONN_SERVER_SWEEP 0x20000000u -#define WT_CONN_WAS_BACKUP 0x40000000u +#define WT_CONN_DEBUG_CURSOR_COPY 0x00000100u +#define WT_CONN_DEBUG_REALLOC_EXACT 0x00000200u +#define WT_CONN_DEBUG_SLOW_CKPT 0x00000400u +#define WT_CONN_EVICTION_NO_LOOKASIDE 0x00000800u +#define WT_CONN_EVICTION_RUN 0x00001000u +#define WT_CONN_INCR_BACKUP 0x00002000u +#define WT_CONN_IN_MEMORY 0x00004000u +#define WT_CONN_LEAK_MEMORY 0x00008000u +#define WT_CONN_LOOKASIDE_OPEN 0x00010000u +#define WT_CONN_LSM_MERGE 0x00020000u +#define WT_CONN_OPTRACK 0x00040000u +#define WT_CONN_PANIC 0x00080000u +#define WT_CONN_READONLY 0x00100000u +#define WT_CONN_RECONFIGURING 0x00200000u +#define WT_CONN_RECOVERING 0x00400000u +#define WT_CONN_SALVAGE 0x00800000u +#define WT_CONN_SERVER_ASYNC 0x01000000u +#define WT_CONN_SERVER_CAPACITY 0x02000000u +#define WT_CONN_SERVER_CHECKPOINT 0x04000000u +#define WT_CONN_SERVER_LOG 0x08000000u +#define WT_CONN_SERVER_LSM 0x10000000u +#define WT_CONN_SERVER_STATISTICS 0x20000000u +#define WT_CONN_SERVER_SWEEP 0x40000000u +#define WT_CONN_WAS_BACKUP 0x80000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/cursor.i b/src/third_party/wiredtiger/src/include/cursor.i index 7d637168ad1..cca55c76207 100644 --- a/src/third_party/wiredtiger/src/include/cursor.i +++ b/src/third_party/wiredtiger/src/include/cursor.i @@ -18,6 +18,20 @@ __cursor_set_recno(WT_CURSOR_BTREE *cbt, uint64_t v) } /* + * __cursor_copy_release -- + * Release memory used by the key and value in cursor copy debug mode. + */ +static inline int +__cursor_copy_release(WT_CURSOR *cursor) +{ + if (F_ISSET(S2C((WT_SESSION_IMPL *)cursor->session), WT_CONN_DEBUG_CURSOR_COPY)) { + WT_RET(__wt_cursor_copy_release_item(cursor, &cursor->key)); + WT_RET(__wt_cursor_copy_release_item(cursor, &cursor->value)); + } + return (0); +} + +/* * __cursor_novalue -- * Release any cached value before an operation that could update the transaction context and * free data a value is pointing to. diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index f3710895938..46de0e10f7f 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -509,6 +509,8 @@ extern int __wt_cursor_cache_release(WT_SESSION_IMPL *session, WT_CURSOR *cursor extern int __wt_cursor_cached(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_compare_notsup(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_copy_release_item(WT_CURSOR *cursor, WT_ITEM *item) + WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_equals(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) diff --git a/src/third_party/wiredtiger/src/include/hardware.h b/src/third_party/wiredtiger/src/include/hardware.h index 9947de8b26a..889756c74e9 100644 --- a/src/third_party/wiredtiger/src/include/hardware.h +++ b/src/third_party/wiredtiger/src/include/hardware.h @@ -16,6 +16,13 @@ (v) = (val); \ } while (0) +/* Write after all previous stores are completed. */ +#define WT_ORDERED_WRITE(v, val) \ + do { \ + WT_WRITE_BARRIER(); \ + (v) = (val); \ + } while (0) + /* * Read a shared location and guarantee that subsequent reads do not see any earlier state. */ diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 7d5c99e7980..9828e74d925 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -131,15 +131,18 @@ if (*(void **)__p != NULL) \ __wt_free_int(session, __p); \ } while (0) + +/* Overwrite whether or not this is a diagnostic build. */ +#define __wt_explicit_overwrite(p, size) memset(p, WT_DEBUG_BYTE, size) #ifdef HAVE_DIAGNOSTIC -#define __wt_overwrite_and_free(session, p) \ - do { \ - memset(p, WT_DEBUG_BYTE, sizeof(*(p))); \ - __wt_free(session, p); \ +#define __wt_overwrite_and_free(session, p) \ + do { \ + __wt_explicit_overwrite(p, sizeof(*(p))); \ + __wt_free(session, p); \ } while (0) #define __wt_overwrite_and_free_len(session, p, len) \ do { \ - memset(p, WT_DEBUG_BYTE, len); \ + __wt_explicit_overwrite(p, len); \ __wt_free(session, p); \ } while (0) #else diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 59d201e5110..50a2d4ce2b1 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -54,14 +54,13 @@ typedef enum { #define WT_TS_MAX UINT64_MAX /* End of time */ /* - * We format timestamps in a couple of ways, declare appropriate sized buffers. - * Hexadecimal is 2x the size of the value. MongoDB format (high/low pairs of - * 4B unsigned integers, with surrounding parenthesis and dividing comma), is - * 2x the maximum digits from a 4B unsigned integer + 3. Both sizes include a - * trailing nul byte as well. + * We format timestamps in a couple of ways, declare appropriate sized buffers. Hexadecimal is 2x + * the size of the value. MongoDB format (high/low pairs of 4B unsigned integers, with surrounding + * parenthesis and separating comma and space), is 2x the maximum digits from a 4B unsigned integer + * plus 4. Both sizes include a trailing nul byte as well. */ #define WT_TS_HEX_STRING_SIZE (2 * sizeof(wt_timestamp_t) + 1) -#define WT_TS_INT_STRING_SIZE (2 * 10 + 3 + 1) +#define WT_TS_INT_STRING_SIZE (2 * 10 + 4 + 1) /* * Perform an operation at the specified isolation level. diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index ddcdeffedf1..702479b797d 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2202,10 +2202,15 @@ struct __wt_connection { * checkpoint_retention, adjust log archiving to retain the log records of this number of * checkpoints. Zero or one means perform normal archiving., an integer between 0 and 1024; * default \c 0.} - * @config{ eviction, if true\, modify internal - * algorithms to change skew to force lookaside eviction to happen more aggressively. This - * includes but is not limited to not skewing newest\, not favoring leaf pages\, and - * modifying the eviction score mechanism., a boolean flag; default \c false.} + * @config{ cursor_copy, if true\, use the system + * allocator to make a copy of any data returned by a cursor operation and return the copy + * instead. The copy is freed on the next cursor operation. This allows memory sanitizers + * to detect inappropriate references to memory owned by cursors., a boolean flag; default + * \c false.} + * @config{ eviction, if true\, modify internal algorithms + * to change skew to force lookaside eviction to happen more aggressively. This includes + * but is not limited to not skewing newest\, not favoring leaf pages\, and modifying the + * eviction score mechanism., a boolean flag; default \c false.} * @config{ realloc_exact, if true\, reallocation of memory will only * provide the exact amount requested. This will help with spotting memory allocation * issues more easily., a boolean flag; default \c false.} @@ -2806,22 +2811,28 @@ struct __wt_connection { * checkpoint_retention, adjust log archiving to retain the log records of this number of * checkpoints. Zero or one means perform normal archiving., an integer between 0 and 1024; default * \c 0.} - * @config{ eviction, if true\, modify internal algorithms to change - * skew to force lookaside eviction to happen more aggressively. This includes but is not limited - * to not skewing newest\, not favoring leaf pages\, and modifying the eviction score mechanism., a - * boolean flag; default \c false.} - * @config{ realloc_exact, if true\, - * reallocation of memory will only provide the exact amount requested. This will help with - * spotting memory allocation issues more easily., a boolean flag; default \c false.} - * @config{ rollback_error, return a WT_ROLLBACK error from a transaction - * operation about every Nth operation to simulate a collision., an integer between 0 and 10M; - * default \c 0.} - * @config{ slow_checkpoint, if true\, slow down checkpoint - * creation by slowing down internal page processing., a boolean flag; default \c false.} - * @config{ table_logging, if true\, write transaction related information to - * the log for all operations\, even operations for tables with logging turned off. This setting - * introduces a log format change that may break older versions of WiredTiger. These operations are - * informational and skipped in recovery., a boolean flag; default \c false.} + * @config{ cursor_copy, if true\, use the system allocator to make a + * copy of any data returned by a cursor operation and return the copy instead. The copy is freed + * on the next cursor operation. This allows memory sanitizers to detect inappropriate references + * to memory owned by cursors., a boolean flag; default \c false.} + * @config{ + * eviction, if true\, modify internal algorithms to change skew to force lookaside eviction to + * happen more aggressively. This includes but is not limited to not skewing newest\, not favoring + * leaf pages\, and modifying the eviction score mechanism., a boolean flag; default \c false.} + * @config{ realloc_exact, if true\, reallocation of memory will only provide + * the exact amount requested. This will help with spotting memory allocation issues more easily., + * a boolean flag; default \c false.} + * @config{ rollback_error, return a + * WT_ROLLBACK error from a transaction operation about every Nth operation to simulate a + * collision., an integer between 0 and 10M; default \c 0.} + * @config{ + * slow_checkpoint, if true\, slow down checkpoint creation by slowing down internal page + * processing., a boolean flag; default \c false.} + * @config{ table_logging, if + * true\, write transaction related information to the log for all operations\, even operations for + * tables with logging turned off. This setting introduces a log format change that may break older + * versions of WiredTiger. These operations are informational and skipped in recovery., a boolean + * flag; default \c false.} * @config{ ),,} * @config{direct_io, Use \c O_DIRECT on POSIX systems\, and \c FILE_FLAG_NO_BUFFERING on Windows to * access files. Options are given as a list\, such as <code>"direct_io=[data]"</code>. Configuring diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 451d4f2faae..5ca567f2646 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -86,10 +86,9 @@ __rec_append_orig_value( /* * If we're saving the original value for a birthmark, transfer over the transaction ID and - * clear out the birthmark update. - * - * Else, set the entry's transaction information to the lowest possible value. Cleared memory - * matches the lowest possible transaction ID and timestamp, do nothing. + * clear out the birthmark update. Else, set the entry's transaction information to the lowest + * possible value (as cleared memory matches the lowest possible transaction ID and timestamp, + * do nothing). */ if (upd->type == WT_UPDATE_BIRTHMARK) { append->txnid = upd->txnid; @@ -100,13 +99,15 @@ __rec_append_orig_value( /* Append the new entry into the update list. */ WT_PUBLISH(upd->next, append); - __wt_cache_page_inmem_incr(session, page, size); + /* Replace the birthmark with an aborted transaction. */ if (upd->type == WT_UPDATE_BIRTHMARK) { - upd->type = WT_UPDATE_STANDARD; - upd->txnid = WT_TXN_ABORTED; + WT_ORDERED_WRITE(upd->txnid, WT_TXN_ABORTED); + WT_ORDERED_WRITE(upd->type, WT_UPDATE_STANDARD); } + __wt_cache_page_inmem_incr(session, page, size); + err: __wt_scr_free(session, &tmp); return (ret); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 5f63c4ff3a6..57b617e53a4 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1723,7 +1723,7 @@ tasks: extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row alter=1 backups=1 compaction=1 data_extend=1 prepare=1 rebalance=1 salvage=1 statistics=1 statistics_server=1 verify=1 - func: "format test" vars: - extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row direct_io=1 firstfit=1 internal_key_truncation=1 + extra_args: checkpoints=1 leak_memory=0 mmap=1 file_type=row firstfit=1 internal_key_truncation=1 - func: "format test" vars: extra_args: leak_memory=0 mmap=1 file_type=row checkpoints=0 in_memory=1 reverse=1 truncate=1 @@ -1954,6 +1954,21 @@ tasks: format_test_script_args: -S times: 16 + - name: race-condition-stress-sanitizer-test + # set an evergreen timeout of 25 hours + exec_timeout_secs: 90000 + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + configure_env_vars: CC="/opt/mongodbtoolchain/v3/bin/clang -fsanitize=address" PATH=/opt/mongodbtoolchain/v3/bin:$PATH CFLAGS="-ggdb -fPIC -fno-omit-frame-pointer" + posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib + - func: "format test script" + vars: + # set a format.sh timeout of 24 hours + format_test_script_args: -R -t 1440 + test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer + - name: recovery-stress-test #set a 25 hours timeout exec_timeout_secs: 90000 @@ -2067,6 +2082,47 @@ tasks: cp ../../../bench/wtperf/split_heavy.wtperf . ./wtperf -O ./split_heavy.wtperf -o verbose=2 + - name: static-wt-build-test + commands: + - func: "get project" + - func: "compile wiredtiger" + vars: + posix_configure_flags: --disable-shared --with-builtins=snappy,zlib + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix" + shell: bash + script: | + set -o errexit + set -o verbose + + # Delete wt util + rm -f wt + # Build static wt util + make CC='eval "g++ -static"' ${smp_command|} + # -V option displays Wiredtiger library version + ./wt -V + + if [ $? -ne 0 ]; then + echo "Error, WT util is not generated or is not functioning" + exit 1 + fi + + ldd wt || wt_static_build=1 + + if [ $wt_static_build -ne 1 ]; then + echo "Error, WT util is not statically linked" + exit 1 + fi + + - name: compatibility-test-for-wiredtiger-releases + commands: + - func: "get project" + - command: subprocess.exec + params: + working_dir: "wiredtiger" + command: bash test/evergreen/compatibility_test_for_wiredtiger_releases.sh + buildvariants: - name: ubuntu1804 display_name: Ubuntu 18.04 @@ -2102,7 +2158,9 @@ buildvariants: - name: split-stress-test - name: format-stress-test - name: format-stress-smoke-test + - name: race-condition-stress-sanitizer-test - name: checkpoint-stress-test + - name: static-wt-build-test - name: ubuntu1804-compilers display_name: Ubuntu 18.04 Compilers @@ -2168,6 +2226,22 @@ buildvariants: - name: unit-test - name: fops - name: time-shift-sensitivity-test + - name: compile-msan + - name: make-check-msan-test + - name: compile-ubsan + - name: ubsan-test + - name: linux-directio + - name: syscall-linux + - name: compile-asan + - name: make-check-asan-test + - name: checkpoint-filetypes-test + - name: unit-test-long + - name: spinlock-gcc-test + - name: spinlock-pthread-adaptive-test + - name: compile-wtperf + - name: wtperf-test + - name: ftruncate-test + - name: long-test - name: large-scale-test display_name: Large scale testing @@ -2184,6 +2258,7 @@ buildvariants: - ubuntu1804-test tasks: - name: compatibility-test-for-mongodb-releases + - name: compatibility-test-for-wiredtiger-releases - name: windows-64 display_name: Windows 64-bit diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_wiredtiger_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_wiredtiger_releases.sh new file mode 100755 index 00000000000..9a571e733b4 --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_wiredtiger_releases.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +############################################################################################## +# Run WiredTiger test/format check against below 3 releases to ensure backward compatibility: +# - current release (develop branch) +# - previous release (WTx.y.z tag) +# - previous previous release (WTa.b.c tag) +############################################################################################## + +set -e + +BUILD_DIR="build_posix" # The relative directory of WiredTiger repo tree + +########################################################################### +# This function will +# - retrieve the previous release tag number based on the count provided +# +# arg1: count of previous release +########################################################################### +get_release() +{ + prev_cnt=$1 + rel=$(git tag | grep -v release | egrep "^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}" | tail -$prev_cnt | head -1) + echo "$rel" +} + +############################################################# +# This function will +# - checkout git tree of the desired release (via arg1) +# - make a build +# - use the generated binary 't' to test the configuration +# +# arg1: release indicator (mapped to release branch/tag) +############################################################# +build_test_db() +{ + rel_ind="$1" + echo "Starting to build and test format for \"$rel_ind\" branch/release ..." + + # Parse the release indicator into release number. + # Checkout the release, and in the case of a branch, refresh it. + case "$rel_ind" in + "develop") # current release + rel="develop" + git checkout $rel + git pull --rebase + ;; + "r1") # previous release + rel="$(get_release 1)" + git checkout $rel + ;; + "r2") # previous previous release + rel="$(get_release 2)" + git checkout $rel + ;; + *) + echo "Unexpected branch/release number: \"$rel_ind\"" + exit 1 + ;; + esac + + echo "Building release: \"$rel\"" + + # Ensure read only testing hasn't left things difficult to cleanup. + chmod -R u+w ${BUILD_DIR} + + # Configure and build + cd ${BUILD_DIR} + sh reconf + ../configure --disable-strict --disable-shared --enable-diagnostic + make -j $(grep -c ^processor /proc/cpuinfo) + + # Test the configuration + cd test/format + cat > CONFIG <<- EOF + reverse=0 + runs=1 + rows=10000 + ops=1000 + threads=2 + evict_max=0 + compression=none + logging_compression=none + encryption=none + EOF + ./t + cd ../.. + + rm -f db + + # Go back to WiredTiger repo directory + cd .. + + # Archive the whole build_posix directory + echo "Archiving ${BUILD_DIR} directory to ${BUILD_DIR}.${rel_ind} ..." + rm -rf ${BUILD_DIR}.${rel_ind} # if dir exists, the next cp command would write it as a sub-dir + cp -R ${BUILD_DIR} ${BUILD_DIR}.${rel_ind} +} + +############################################################# +# This function will +# - go into the archived build directory for the release +# - use the 'wt' binary to cross-check the wt URI +# +# arg1: release indicator (mapped to release branch/tag) +############################################################# +verify_uri() +{ + rel_ind="$1" + echo "Starting to verify URI for \"$rel_ind\" branch/release ..." + + # Go into the archived build directory of the release + cd ${BUILD_DIR}.${rel_ind} + + dir=test/format/RUNDIR + + # Along the way, we changed the running configuration from + # RUNDIR/run to RUNDIR/CONFIG. Check for both. Once that + # change propogates back, we can just use $dir/CONFIG. + if test -e $dir/CONFIG; then + cfile=$dir/CONFIG + else + cfile=$dir/run + fi + isfile=`grep data_source $cfile | grep -c file || exit 0` + if test "$isfile" -ne 0; then + uri="file:wt" + else + uri="table:wt" + fi + + # Cross-check the wt URI against the build directory + case "$rel_ind" in + "develop") # Test directory of current release using wt binaries from "all 3 releases" + echo "Verifying $uri using $devwt binary" + # Remove basecfg as the statistic logging block + rm -f $dir/WiredTiger.basecfg + $devwt -h $dir verify $uri + # We don't guarantee that the wiredtiger_open config is backwards + # and forwards compatible. Just delete it for now. + echo "Verifying $uri using $r1wt binary" + $r1wt -h $dir verify $uri + echo "Verifying $uri using $r2wt binary" + $r2wt -h $dir verify $uri + ;; + "r1") # Test directory of previous release using wt binaries from both "current" and "previous" releases + rm -f $dir/WiredTiger.basecfg + echo "Verifying $uri using $r1wt binary" + $r1wt -h $dir verify $uri + echo "Verifying $uri using $devwt binary" + $devwt -h $dir verify $uri + ;; + "r2") # Test directory of previous previous release using wt binaries from both "current" and "previous previous" releases + rm -f $dir/WiredTiger.basecfg + echo "Verifying $uri using $r2wt binary" + $r2wt -h $dir verify $uri + echo "Verifying $uri using $devwt binary" + $devwt -h $dir verify $uri + ;; + *) + echo "Unexpected argument value" + ;; + esac + + # Go back to WiredTiger repo directory + cd .. +} + +# Firstly, do some cleanup for the previous run. +git reset --hard && git clean -fdqx -e '*.tgz' + +# Build and test format, then set the archived wt binary for each release +build_test_db "develop" +devwt=../${BUILD_DIR}.develop/wt +build_test_db "r1" +r1wt=../${BUILD_DIR}.r1/wt +build_test_db "r2" +r2wt=../${BUILD_DIR}.r2/wt + +# Cross-check the URI for the 3 releases +verify_uri "develop" +verify_uri "r1" +verify_uri "r2" + diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index f9baf495a83..36a676e58da 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -27,6 +27,7 @@ usage() { echo " -h home run directory (defaults to .)" echo " -j parallel jobs to execute in parallel (defaults to 8)" echo " -n total total jobs to execute (defaults to no limit)" + echo " -R run timing stress split test configurations (defaults to off)" echo " -S run smoke-test configurations (defaults to off)" echo " -t minutes minutes to run (defaults to no limit)" echo " -v verbose output (defaults to off)" @@ -70,6 +71,7 @@ home="." minutes=0 parallel_jobs=8 smoke_test=0 +timing_stress_split_test=0 total_jobs=0 verbose=0 format_binary="./t" @@ -105,6 +107,9 @@ while :; do exit 1 } shift ; shift ;; + -R) + timing_stress_split_test=1 + shift ;; -S) smoke_test=1 shift ;; @@ -362,6 +367,12 @@ format() args=${smoke_list[$smoke_next]} smoke_next=$(($smoke_next + 1)) echo "$name: starting smoke-test job in $dir ($(date))" + elif [[ $timing_stress_split_test -ne 0 ]]; then + args=$format_args + for k in {1..7}; do + args+=" timing_stress_split_$k=$(($RANDOM%2))" + done + echo "$name: starting timing-stress-split job in $dir ($(date))" else args=$format_args diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index 0f86d5b4560..91522287a6f 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -95,10 +95,8 @@ for d in os.listdir(wt_3rdpartydir): sys.path.insert(1, os.path.join(wt_3rdpartydir, d, subdir)) break -import wttest -# Use the same version of unittest found by wttest.py -unittest = wttest.unittest -from testscenarios.scenarios import generate_scenarios +# unittest will be imported later, near to when it is needed. +unittest = None def usage(): print('Usage:\n\ @@ -106,6 +104,7 @@ def usage(): $ python ../test/suite/run.py [ options ] [ tests ]\n\ \n\ Options:\n\ + --asan run with an ASAN enabled shared library\n\ -C file | --configcreate file create a config file for controlling tests\n\ -c file | --config file use a config file for controlling tests\n\ -D dir | --dir dir use dir rather than WT_TEST.\n\ @@ -133,6 +132,32 @@ Tests:\n\ When -s is present, there must be a test named.\n\ ') +# Find an executable of the given name in the execution path. +def which(name): + path = os.getenv('PATH') + for pathdir in path.split(os.path.pathsep): + fname = os.path.join(pathdir, name) + if os.path.exists(fname) and os.access(fname, os.X_OK): + return fname + return None + +# Follow a symbolic link, returning the target +def follow_symlinks(pathname): + return os.path.realpath(pathname) + +# Find all instances of a filename under a directory +def find(topdir, filename): + results = [] + for root, dirs, files in os.walk(topdir, followlinks=True): + if filename in files: + results.append(os.path.join(root, filename)) + return results + +# Show an environment variable if verbose enough. +def show_env(verbose, envvar): + if verbose >= 2: + print(envvar + "=" + os.getenv(envvar)) + # capture the category (AKA 'subsuite') part of a test name, # e.g. test_util03 -> util reCatname = re.compile(r"test_([^0-9]+)[0-9]*") @@ -266,11 +291,14 @@ def testsFromArg(tests, loader, arg, scenario): for t in xrange(start, end+1): addScenarioTests(tests, loader, 'test%03d' % t, scenario) -if __name__ == '__main__': - tests = unittest.TestSuite() +def error(exitval, prefix, msg): + print('*** ERROR: {}: {}'.format(prefix, msg.replace('\n', '\n*** '))) + sys.exit(exitval) +if __name__ == '__main__': # Turn numbers and ranges into test module names preserve = timestamp = debug = dryRun = gdbSub = lldbSub = longtest = ignoreStdout = False + asan = False parallel = 0 random_sample = 0 configfile = None @@ -287,6 +315,9 @@ if __name__ == '__main__': # Command line options if arg[0] == '-': option = arg[1:] + if option == '-asan': + asan = True + continue if option == '-dir' or option == 'D': if dirarg != None or len(args) == 0: usage() @@ -369,6 +400,83 @@ if __name__ == '__main__': sys.exit(2) testargs.append(arg) + if asan: + # To run ASAN, we need to ensure these environment variables are set: + # ASAN_SYMBOLIZER_PATH full path to the llvm-symbolizer program + # LD_LIBRARY_PATH includes path with wiredtiger shared object + # LD_PRELOAD includes the ASAN runtime library + # + # Note that LD_LIBRARY_PATH has already been set above. The trouble with + # simply setting these variables in the Python environment is that it's + # too late. LD_LIBRARY_PATH is commonly cached by the shared library + # loader at program startup, and that's already been done before Python + # begins execution. Likewise, any preloading indicated by LD_PRELOAD + # has already been done. + # + # Our solution is to set the variables as appropriate, and then restart + # Python with the same argument list. The shared library loader will + # have everything it needs on the second go round. + # + # Note: If the ASAN stops the program with the error: + # Shadow memory range interleaves with an existing memory mapping. + # ASan cannot proceed correctly. + # + # try rebuilding with the clang options: + # "-mllvm -asan-force-dynamic-shadow=1" + # and make sure that clang is used for all compiles. + # + # We'd like to show this as a message, but there's no good way to + # detect this error from here short of capturing/parsing all output + # from the test run. + ASAN_ENV = "__WT_TEST_SUITE_ASAN" # if set, we've been here before + ASAN_SYMBOLIZER_PROG = "llvm-symbolizer" + ASAN_SYMBOLIZER_ENV = "ASAN_SYMBOLIZER_PATH" + LD_PRELOAD_ENV = "LD_PRELOAD" + SO_FILE_NAME = "libclang_rt.asan-x86_64.so" + if not os.environ.get(ASAN_ENV): + if verbose >= 2: + print('Enabling ASAN environment and rerunning python') + os.environ[ASAN_ENV] = "1" + show_env(verbose, "LD_LIBRARY_PATH") + if not os.environ.get(ASAN_SYMBOLIZER_ENV): + os.environ[ASAN_SYMBOLIZER_ENV] = which(ASAN_SYMBOLIZER_PROG) + if not os.environ.get(ASAN_SYMBOLIZER_ENV): + error(ASAN_SYMBOLIZER_ENV, + 'symbolizer program not found in PATH') + show_env(verbose, ASAN_SYMBOLIZER_ENV) + if not os.environ.get(LD_PRELOAD_ENV): + symbolizer = follow_symlinks(os.environ[ASAN_SYMBOLIZER_ENV]) + bindir = os.path.dirname(symbolizer) + sofiles = [] + if os.path.basename(bindir) == 'bin': + libdir = os.path.join(os.path.dirname(bindir), 'lib') + sofiles = find(libdir, SO_FILE_NAME) + if len(sofiles) != 1: + if len(sofiles) == 0: + fmt = 'ASAN shared library file not found.\n' + \ + 'Set {} to the file location and rerun.' + error(3, SO_FILE_NAME, fmt.format(LD_PRELOAD_ENV)) + else: + fmt = 'multiple ASAN shared library files found\n' + \ + 'under {}, expected just one.\n' + \ + 'Set {} to the correct file location and rerun.' + error(3, SO_FILE_NAME, fmt.format(libdir, LD_PRELOAD_ENV)) + os.environ[LD_PRELOAD_ENV] = sofiles[0] + show_env(verbose, LD_PRELOAD_ENV) + + # Restart python! + python = sys.executable + os.execl(python, python, *sys.argv) + elif verbose >= 2: + print('Python restarted for ASAN') + + # We don't import wttest until after ASAN environment variables are set. + import wttest + # Use the same version of unittest found by wttest.py + unittest = wttest.unittest + tests = unittest.TestSuite() + from testscenarios.scenarios import generate_scenarios + # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, lldbSub, diff --git a/src/third_party/wiredtiger/test/suite/test_bug018.py b/src/third_party/wiredtiger/test/suite/test_bug018.py index 5e5472f6eef..53294bcf4d8 100755 --- a/src/third_party/wiredtiger/test/suite/test_bug018.py +++ b/src/third_party/wiredtiger/test/suite/test_bug018.py @@ -106,14 +106,14 @@ class test_bug018(wttest.WiredTigerTestCase, suite_subprocess): # It's possible the second table can't even be opened. # That can happen only if the root page was not pushed out. - # So if we get an error, make sure we're getting the right - # error message. - - self.captureerr.check(self) # check error messages until now + # We can't depend on the text of a particular error message to be + # emitted, so we'll just ignore the error. + self.captureerr.check(self) # check there is no error output so far try: results2 = list(self.session.open_cursor(self.uri2)) except: - self.captureerr.checkAdditionalPattern(self, 'unable to read root page') + # Make sure there's some error, but we don't care what. + self.captureerr.checkAdditionalPattern(self, '.') results2 = [] self.assertEqual(results1, results2) diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode08.py b/src/third_party/wiredtiger/test/suite/test_debug_mode08.py new file mode 100755 index 00000000000..0794d34e494 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode08.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest, wiredtiger, test_base03 + +# test_debug_mode08.py +# Test the debug mode setting for cursor_copy. +# There's no great way to detect that cursor_copy is really working, as +# the extra malloc/free that WT does cannot easily be detected. However, +# it's useful to run some tests with the mode enabled. We make a subclass +# of test_base03, so we'll inherit those tests to be run with the debug mode +# configuration enabled. +class test_debug_mode08(test_base03.test_base03): + conn_config = 'debug_mode=(cursor_copy=true)' + uri = 'file:test_debug_mode08' + + def test_reconfig(self): + ''' Test reconfigure with some minimal cursor activity. ''' + self.session.create(self.uri, 'key_format=s,value_format=s') + cursor = self.session.open_cursor(self.uri, None) + cursor['key'] = 'value' + cursor.close() + + conn_reconfig = 'debug_mode=(cursor_copy=false)' + self.conn.reconfigure(conn_reconfig) + cursor = self.session.open_cursor(self.uri, None) + cursor['key'] = 'value' + cursor.close() + + self.conn.reconfigure(self.conn_config) + cursor = self.session.open_cursor(self.uri, None) + cursor['key'] = 'value' + cursor.close() + +if __name__ == '__main__': + wttest.run() |