diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2020-12-21 13:34:10 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-12-21 02:57:48 +0000 |
| commit | 1fe4cdec74edfb1ede2d56fc4fd61cf38da6334e (patch) | |
| tree | f6f9120e9c1cb940045d877ddae1982b3a591be3 /src/third_party/wiredtiger | |
| parent | 7c8e5d2492949658297e4def62c0428823585cd1 (diff) | |
| download | mongo-1fe4cdec74edfb1ede2d56fc4fd61cf38da6334e.tar.gz | |
Import wiredtiger: 1fcb0f1e1d84c105a07588ca4be3f2e6fd25b2d4 from branch mongodb-4.4
ref: d0d32d917b..1fcb0f1e1d
for: 4.4.3
WT-6432 Add test case to misuse the timestamp API
WT-6740 Fix unintentionally releasing the snapshot by mistreating the recovery session as an eviction session
WT-6820 Use WT_TIME_WINDOW in hs_rec.c
WT-6831 Retry search if it race with prepared update commit/rollback
WT-6846 Initial test program for the new cpp test framework
WT-6861 Add the ability to log messages about unexpected timestamp usage
WT-6983 Make wiredtiger.in text wrapping consistent across Python versions
WT-6992 Add timing stress between datastore and history store search
WT-6994 Dump the cursor page whenever a key out order is detected
Diffstat (limited to 'src/third_party/wiredtiger')
46 files changed, 1192 insertions, 422 deletions
diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index da149c335f4..117e54bb613 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -29,6 +29,7 @@ examples/c # Test programs. test/bloom test/checkpoint +test/cppsuite HAVE_CXX test/csuite test/cursor_order test/fops diff --git a/src/third_party/wiredtiger/dist/api_config.py b/src/third_party/wiredtiger/dist/api_config.py index ecaaa065dd2..3ab0ec15504 100755 --- a/src/third_party/wiredtiger/dist/api_config.py +++ b/src/third_party/wiredtiger/dist/api_config.py @@ -132,7 +132,7 @@ for line in open(f, 'r'): fix_sentence_endings=True) # Separate at spaces, and after a set of non-breaking space indicators. w.wordsep_re = w.wordsep_simple_re = \ - re.compile(r'(\s+|(?<= )[\w_,.;:]*)') + re.compile(r'(\s+|(?<= )[\w_,.;:]+)') for c in api_data.methods[config_name].config: if 'undoc' in c.flags: continue diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index c76938f7919..957ef40ff09 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -45,6 +45,43 @@ class Config: common_runtime_config = [ Config('app_metadata', '', r''' application-owned metadata for this object'''), + Config('assert', '', r''' + enable enhanced checking. ''', + type='category', subconfig= [ + Config('write_timestamp', 'off', r''' + verify that commit timestamps are used per the configured + \c write_timestamp_usage option for this table''', + choices=['off', 'on']), + Config('read_timestamp', 'none', r''' + verify that timestamps should \c always or \c never be used + on reads with this table. Verification is \c none + if mixed read use is allowed''', + choices=['always', 'never', 'none']) + ], undoc=True), + Config('verbose', '[]', r''' + enable messages for various events. Options are given as a + list, such as <code>"verbose=[write_timestamp]"</code>''', + type='list', choices=[ + 'write_timestamp', + ]), + Config('write_timestamp_usage', 'none', r''' + describe how timestamps are expected to be used on modifications to + the table. This option should be used in conjunction with the + corresponding \c write_timestamp configuration under the \c assert and + \c verbose options to provide logging and assertions for incorrect + timestamp usage. The choices are \c always which ensures a timestamp is + used for every operation on a table, \c key_consistent to ensure that + once timestamps are used for a key, they are always used, \c ordered is + like \c key_consistent except it also enforces that subsequent updates + to each key must use increasing timestamps, \c mixed_mode is like + \c ordered except that updates with no timestamp are allowed and have + the effect of resetting the chain of updates once the transaction ID + based snapshot is no longer relevant, \c never enforces that timestamps + are never used for a table and \c none does not enforce any expectation + on timestamp usage meaning that no log message or assertions will be + produced regardless of the corresponding \c assert and \c verbose + settings''', + choices=['always', 'key_consistent', 'mixed_mode', 'never', 'none', 'ordered']), ] # Metadata shared by all schema objects @@ -169,26 +206,6 @@ file_runtime_config = common_runtime_config + [ option leads to an advisory call to an appropriate operating system API where available''', choices=['none', 'random', 'sequential']), - Config('assert', '', r''' - enable enhanced checking. ''', - type='category', subconfig= [ - Config('commit_timestamp', 'none', r''' - verify that timestamps should 'always' or 'never' be used - on modifications with this table. Verification is 'none' - if mixed update use is allowed. If 'key_consistent' is - set then all updates to a specific key must be the same - with respect to timestamp usage or not.''', - choices=['always', 'key_consistent', 'never', 'none']), - Config('durable_timestamp', 'none', r''' - verify that durable timestamps should 'always' or 'never' be used - on modifications with this table.''', - choices=['always', 'key_consistent', 'never', 'none']), - Config('read_timestamp', 'none', r''' - verify that timestamps should 'always' or 'never' be used - on reads with this table. Verification is 'none' - if mixed read use is allowed.''', - choices=['always', 'never', 'none']) - ], undoc=True), Config('cache_resident', 'false', r''' do not ever evict the object's pages from cache. Not compatible with LSM tables; see @ref tuning_cache_resident for more information''', @@ -695,9 +712,9 @@ connection_runtime_config = [ type='list', undoc=True, choices=[ 'aggressive_sweep', 'backup_rename', 'checkpoint_slow', 'history_store_checkpoint_delay', - 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', 'split_2', - 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']), - Config('verbose', '', r''' + 'history_store_search', 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', + 'split_2', 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']), + Config('verbose', '[]', r''' enable messages for various events. Options are given as a list, such as <code>"verbose=[evictserver,read]"</code>''', type='list', choices=[ diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 51205582206..f7578bb1ff4 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "d0d32d917bf0137bb89477a7b22fc265bc08dfa8" + "commit": "1fcb0f1e1d84c105a07588ca4be3f2e6fd25b2d4" } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 00ae7165454..1801394cf27 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -444,6 +444,8 @@ __cursor_key_order_check_col(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo return (0); } + WT_RET(__wt_msg(session, "dumping the cursor page")); + WT_RET(__wt_debug_cursor_page(&cbt->iface, NULL)); WT_RET_PANIC(session, EINVAL, "WT_CURSOR.%s out-of-order returns: returned key %" PRIu64 " then key %" PRIu64, next ? "next" : "prev", cbt->lastrecno, cbt->recno); @@ -476,6 +478,8 @@ __cursor_key_order_check_row(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo WT_ERR(__wt_scr_alloc(session, 512, &a)); WT_ERR(__wt_scr_alloc(session, 512, &b)); + WT_ERR(__wt_msg(session, "dumping the cursor page")); + WT_ERR(__wt_debug_cursor_page(&cbt->iface, NULL)); WT_ERR_PANIC(session, EINVAL, "WT_CURSOR.%s out-of-order returns: returned key %.1024s then key %.1024s", next ? "next" : "prev", diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index e5e2efada27..23a9c4f8078 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -921,7 +921,7 @@ __curfile_update_check(WT_CURSOR_BTREE *cbt) else if (btree->type != BTREE_COL_VAR) return (0); - return (__wt_txn_update_check(session, cbt, upd)); + return (__wt_txn_update_check(session, cbt, upd, NULL)); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 60cde6ce8b5..c745a8078ea 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -408,34 +408,6 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) else btree->checksum = CKSUM_UNCOMPRESSED; - /* Debugging information */ - WT_RET(__wt_config_gets(session, cfg, "assert.commit_timestamp", &cval)); - btree->assert_flags = 0; - if (WT_STRING_MATCH("always", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_COMMIT_TS_ALWAYS); - else if (WT_STRING_MATCH("key_consistent", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_COMMIT_TS_KEYS); - else if (WT_STRING_MATCH("never", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_COMMIT_TS_NEVER); - - /* - * A durable timestamp always implies a commit timestamp. But never having a durable timestamp - * does not imply anything about a commit timestamp. - */ - WT_RET(__wt_config_gets(session, cfg, "assert.durable_timestamp", &cval)); - if (WT_STRING_MATCH("always", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_COMMIT_TS_ALWAYS | WT_ASSERT_DURABLE_TS_ALWAYS); - else if (WT_STRING_MATCH("key_consistent", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_DURABLE_TS_KEYS); - else if (WT_STRING_MATCH("never", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_DURABLE_TS_NEVER); - - WT_RET(__wt_config_gets(session, cfg, "assert.read_timestamp", &cval)); - if (WT_STRING_MATCH("always", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_READ_TS_ALWAYS); - else if (WT_STRING_MATCH("never", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_READ_TS_NEVER); - /* Huffman encoding */ WT_RET(__wt_btree_huffman_open(session)); diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index 49e57bd51d8..6be6ea9589c 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -27,6 +27,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_PAGE_MODIFY *mod; WT_SESSION_IMPL *session; WT_UPDATE *old_upd, *upd; + wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; u_int i, skipdepth; bool append, logged; @@ -36,6 +37,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U page = cbt->ref->page; session = CUR2S(cbt); upd = upd_arg; + prev_upd_ts = WT_TS_NONE; append = logged = false; /* @@ -125,10 +127,13 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U old_upd = cbt->ins->upd; if (upd_arg == NULL) { /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd)); + WT_ERR(__wt_txn_update_check(session, cbt, old_upd, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); +#ifdef HAVE_DIAGNOSTIC + upd->prev_durable_ts = prev_upd_ts; +#endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; } else { diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index 652c59c71a0..ef600f7d88c 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -51,6 +51,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, WT_PAGE_MODIFY *mod; WT_SESSION_IMPL *session; WT_UPDATE *last_upd, *old_upd, *upd, **upd_entry; + wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; uint32_t ins_slot; u_int i, skipdepth; @@ -61,6 +62,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, session = CUR2S(cbt); last_upd = NULL; upd = upd_arg; + prev_upd_ts = WT_TS_NONE; inserted_to_update_chain = logged = false; /* @@ -103,10 +105,13 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, if (upd_arg == NULL) { /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd = *upd_entry)); + WT_ERR(__wt_txn_update_check(session, cbt, old_upd = *upd_entry, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); +#ifdef HAVE_DIAGNOSTIC + upd->prev_durable_ts = prev_upd_ts; +#endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 55c59fee565..ef434fa3e5d 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -125,9 +125,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," - "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," - "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"history_store_search\",\"history_store_sweep_race\"," + "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" + ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"verbose", "list", NULL, "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," @@ -154,15 +154,8 @@ static const WT_CONFIG_CHECK confchk_WT_CURSOR_reconfigure[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_assert_subconfigs[] = { - {"commit_timestamp", "string", NULL, - "choices=[\"always\",\"key_consistent\",\"never\"," - "\"none\"]", - NULL, 0}, - {"durable_timestamp", "string", NULL, - "choices=[\"always\",\"key_consistent\",\"never\"," - "\"none\"]", - NULL, 0}, {"read_timestamp", "string", NULL, "choices=[\"always\",\"never\",\"none\"]", NULL, 0}, + {"write_timestamp", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_create_log_subconfigs[] = { @@ -171,12 +164,18 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create_log_subconfigs[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"exclusive_refreshed", "boolean", NULL, NULL, NULL, 0}, {"log", "category", NULL, NULL, confchk_WT_SESSION_create_log_subconfigs, 1}, {"os_cache_dirty_max", "int", NULL, "min=0", NULL, 0}, - {"os_cache_max", "int", NULL, "min=0", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"os_cache_max", "int", NULL, "min=0", NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_begin_transaction_roundup_timestamps_subconfigs[] = {{"prepared", "boolean", NULL, NULL, NULL, 0}, {"read", "boolean", NULL, NULL, NULL, 0}, @@ -241,7 +240,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, @@ -276,6 +275,11 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_drop[] = { @@ -367,15 +371,22 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_verify[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_colgroup_meta[] = { - {"app_metadata", "string", NULL, NULL, NULL, 0}, {"collator", "string", NULL, NULL, NULL, 0}, - {"columns", "list", NULL, NULL, NULL, 0}, {"source", "string", NULL, NULL, NULL, 0}, - {"type", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"app_metadata", "string", NULL, NULL, NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, + {"source", "string", NULL, NULL, NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_file_config[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, @@ -406,13 +417,18 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_file_meta[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, @@ -445,22 +461,35 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, - {"version", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"version", "string", NULL, NULL, NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_index_meta[] = { - {"app_metadata", "string", NULL, NULL, NULL, 0}, {"collator", "string", NULL, NULL, NULL, 0}, - {"columns", "list", NULL, NULL, NULL, 0}, {"extractor", "string", NULL, NULL, NULL, 0}, - {"immutable", "boolean", NULL, NULL, NULL, 0}, {"index_key_columns", "int", NULL, NULL, NULL, 0}, + {"app_metadata", "string", NULL, NULL, NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, + {"extractor", "string", NULL, NULL, NULL, 0}, {"immutable", "boolean", NULL, NULL, NULL, 0}, + {"index_key_columns", "int", NULL, NULL, NULL, 0}, {"key_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"source", "string", NULL, NULL, NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, @@ -493,13 +522,25 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_table_meta[] = { - {"app_metadata", "string", NULL, NULL, NULL, 0}, {"colgroups", "list", NULL, NULL, NULL, 0}, - {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, + {"app_metadata", "string", NULL, NULL, NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"colgroups", "list", NULL, NULL, NULL, 0}, {"collator", "string", NULL, NULL, NULL, 0}, + {"columns", "list", NULL, NULL, NULL, 0}, {"key_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_compatibility_subconfigs[] = { @@ -586,9 +627,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," - "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," - "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"history_store_search\",\"history_store_sweep_race\"," + "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" + ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -662,9 +703,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," - "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," - "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"history_store_search\",\"history_store_sweep_race\"," + "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" + ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -735,9 +776,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," - "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," - "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"history_store_search\",\"history_store_sweep_race\"," + "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" + ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -806,9 +847,9 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"timing_stress_for_test", "list", NULL, "choices=[\"aggressive_sweep\",\"backup_rename\"," "\"checkpoint_slow\",\"history_store_checkpoint_delay\"," - "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," - "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"history_store_search\",\"history_store_sweep_race\"," + "\"prepare_checkpoint_delay\",\"split_1\",\"split_2\",\"split_3\"" + ",\"split_4\",\"split_5\",\"split_6\",\"split_7\",\"split_8\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -865,7 +906,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(json=false,on_close=false," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,verbose=", + "timing_stress_for_test=,verbose=[]", confchk_WT_CONNECTION_reconfigure, 27}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", @@ -876,11 +917,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_CURSOR.reconfigure", "append=false,overwrite=true", confchk_WT_CURSOR_reconfigure, 2}, {"WT_SESSION.alter", "access_pattern_hint=none,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),cache_resident=false," - "exclusive_refreshed=true,log=(enabled=true),os_cache_dirty_max=0" - ",os_cache_max=0", - confchk_WT_SESSION_alter, 8}, + "assert=(read_timestamp=none,write_timestamp=off)," + "cache_resident=false,exclusive_refreshed=true,log=(enabled=true)" + ",os_cache_dirty_max=0,os_cache_max=0,verbose=[]," + "write_timestamp_usage=none", + confchk_WT_SESSION_alter, 10}, {"WT_SESSION.begin_transaction", "ignore_prepare=false,isolation=,name=,operation_timeout_ms=0," "priority=0,read_before_oldest=false,read_timestamp=," @@ -896,14 +937,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.compact", "timeout=1200", confchk_WT_SESSION_compact, 1}, {"WT_SESSION.create", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checksum=uncompressed,colgroups=,collator=," - "columns=,dictionary=0,encryption=(keyid=,name=),exclusive=false," - "extractor=,format=btree,huffman_value=," - "ignore_in_memory_cache_size=false,immutable=false," - "import=(enabled=false,file_metadata=,repair=false)," - "internal_item_max=0,internal_key_max=0," + "assert=(read_timestamp=none,write_timestamp=off)," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,colgroups=,collator=,columns=,dictionary=0" + ",encryption=(keyid=,name=),exclusive=false,extractor=," + "format=btree,huffman_value=,ignore_in_memory_cache_size=false," + "immutable=false,import=(enabled=false,file_metadata=," + "repair=false),internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," @@ -914,8 +954,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," "prefix_compression=false,prefix_compression_min=4,source=," "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "type=file,value_format=u", - confchk_WT_SESSION_create, 44}, + "type=file,value_format=u,verbose=[],write_timestamp_usage=none", + confchk_WT_SESSION_create, 46}, {"WT_SESSION.drop", "checkpoint_wait=true,force=false,lock_wait=true," "remove_files=true", @@ -958,69 +998,75 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "dump_offsets=,dump_pages=false,stable_timestamp=false," "strict=false", confchk_WT_SESSION_verify, 7}, - {"colgroup.meta", "app_metadata=,collator=,columns=,source=,type=file", confchk_colgroup_meta, 5}, + {"colgroup.meta", + "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "collator=,columns=,source=,type=file,verbose=[]," + "write_timestamp_usage=none", + confchk_colgroup_meta, 8}, {"file.config", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checksum=uncompressed,collator=,columns=," - "dictionary=0,encryption=(keyid=,name=),format=btree," - "huffman_value=,ignore_in_memory_cache_size=false," - "internal_item_max=0,internal_key_max=0," - "internal_key_truncate=true,internal_page_max=4KB,key_format=u," - "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," - "leaf_value_max=0,log=(enabled=true),memory_page_image_max=0," - "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "value_format=u", - confchk_file_config, 36}, + "assert=(read_timestamp=none,write_timestamp=off)," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_value=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," + "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," + "log=(enabled=true),memory_page_image_max=0,memory_page_max=5MB," + "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," + "prefix_compression_min=4,split_deepen_min_child=0," + "split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]," + "write_timestamp_usage=none", + confchk_file_config, 38}, {"file.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checkpoint=,checkpoint_backup_info=," - "checkpoint_lsn=,checksum=uncompressed,collator=,columns=," - "dictionary=0,encryption=(keyid=,name=),format=btree," - "huffman_value=,id=,ignore_in_memory_cache_size=false," - "internal_item_max=0,internal_key_max=0," - "internal_key_truncate=true,internal_page_max=4KB,key_format=u," - "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," - "leaf_value_max=0,log=(enabled=true),memory_page_image_max=0," - "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "value_format=u,version=(major=0,minor=0)", - confchk_file_meta, 41}, + "assert=(read_timestamp=none,write_timestamp=off)," + "block_allocation=best,block_compressor=,cache_resident=false," + "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=," + "checksum=uncompressed,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_value=,id=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," + "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," + "log=(enabled=true),memory_page_image_max=0,memory_page_max=5MB," + "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," + "prefix_compression_min=4,split_deepen_min_child=0," + "split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]," + "version=(major=0,minor=0),write_timestamp_usage=none", + confchk_file_meta, 43}, {"index.meta", - "app_metadata=,collator=,columns=,extractor=,immutable=false," - "index_key_columns=,key_format=u,source=,type=file,value_format=u", - confchk_index_meta, 10}, + "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "collator=,columns=,extractor=,immutable=false,index_key_columns=" + ",key_format=u,source=,type=file,value_format=u,verbose=[]," + "write_timestamp_usage=none", + confchk_index_meta, 13}, {"lsm.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checksum=uncompressed,chunks=,collator=," - "columns=,dictionary=0,encryption=(keyid=,name=),format=btree," - "huffman_value=,ignore_in_memory_cache_size=false," - "internal_item_max=0,internal_key_max=0," - "internal_key_truncate=true,internal_page_max=4KB,key_format=u," - "key_gap=10,last=,leaf_item_max=0,leaf_key_max=0," - "leaf_page_max=32KB,leaf_value_max=0,log=(enabled=true)," - "lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16," - "bloom_config=,bloom_hash_count=8,bloom_oldest=false," - "chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB," - "merge_custom=(prefix=,start_generation=0,suffix=),merge_max=15," - "merge_min=0),memory_page_image_max=0,memory_page_max=5MB," - "old_chunks=,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "value_format=u", - confchk_lsm_meta, 40}, + "assert=(read_timestamp=none,write_timestamp=off)," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,chunks=,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_value=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,last=," + "leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," + "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," + "bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8," + "bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB," + "chunk_size=10MB,merge_custom=(prefix=,start_generation=0," + "suffix=),merge_max=15,merge_min=0),memory_page_image_max=0," + "memory_page_max=5MB,old_chunks=,os_cache_dirty_max=0," + "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" + ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "value_format=u,verbose=[],write_timestamp_usage=none", + confchk_lsm_meta, 42}, {"table.meta", - "app_metadata=,colgroups=,collator=,columns=,key_format=u," - "value_format=u", - confchk_table_meta, 6}, + "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "colgroups=,collator=,columns=,key_format=u,value_format=u," + "verbose=[],write_timestamp_usage=none", + confchk_table_meta, 9}, {"wiredtiger_open", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," @@ -1051,7 +1097,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," - "verbose=,verify_metadata=false,write_through=", + "verbose=[],verify_metadata=false,write_through=", confchk_wiredtiger_open, 55}, {"wiredtiger_open_all", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" @@ -1083,7 +1129,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," - "verbose=,verify_metadata=false,version=(major=0,minor=0)," + "verbose=[],verify_metadata=false,version=(major=0,minor=0)," "write_through=", confchk_wiredtiger_open_all, 56}, {"wiredtiger_open_basecfg", @@ -1114,7 +1160,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=,verify_metadata=false,version=(major=0," + "method=fsync),verbose=[],verify_metadata=false,version=(major=0," "minor=0),write_through=", confchk_wiredtiger_open_basecfg, 50}, {"wiredtiger_open_usercfg", @@ -1145,7 +1191,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=,verify_metadata=false,write_through=", + "method=fsync),verbose=[],verify_metadata=false,write_through=", confchk_wiredtiger_open_usercfg, 49}, {NULL, NULL, NULL, 0}}; diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 9b5e2394b61..1d792d74163 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1834,12 +1834,6 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) return (0); } -/* Simple structure for name and flag configuration searches. */ -typedef struct { - const char *name; - uint64_t flag; -} WT_NAME_FLAG; - /* * __wt_verbose_config -- * Set verbose configuration. @@ -1993,6 +1987,7 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) {"backup_rename", WT_TIMING_STRESS_BACKUP_RENAME}, {"checkpoint_slow", WT_TIMING_STRESS_CHECKPOINT_SLOW}, {"history_store_checkpoint_delay", WT_TIMING_STRESS_HS_CHECKPOINT_DELAY}, + {"history_store_search", WT_TIMING_STRESS_HS_SEARCH}, {"history_store_sweep_race", WT_TIMING_STRESS_HS_SWEEP}, {"prepare_checkpoint_delay", WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY}, {"split_1", WT_TIMING_STRESS_SPLIT_1}, {"split_2", WT_TIMING_STRESS_SPLIT_2}, diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 16a9d7812b5..44b2d86f7c5 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -410,6 +410,57 @@ err: } /* + * __conn_dhandle_config_parse -- + * Parse out configuration settings relevant for the data handle. + */ +static int +__conn_dhandle_config_parse(WT_SESSION_IMPL *session) +{ + WT_CONFIG_ITEM cval, sval; + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; + const char **cfg; + + dhandle = session->dhandle; + cfg = dhandle->cfg; + + /* Clear all the flags. */ + dhandle->ts_flags = 0; + + /* Debugging information */ + WT_RET(__wt_config_gets(session, cfg, "assert.write_timestamp", &cval)); + if (WT_STRING_MATCH("on", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_WRITE); + + WT_RET(__wt_config_gets(session, cfg, "assert.read_timestamp", &cval)); + if (WT_STRING_MATCH("always", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_ALWAYS); + else if (WT_STRING_MATCH("never", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_NEVER); + + /* Setup verbose options */ + WT_RET(__wt_config_gets(session, cfg, "verbose", &cval)); + if ((ret = __wt_config_subgets(session, &cval, "write_timestamp", &sval)) == 0 && sval.val != 0) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_VERB_TS_WRITE); + WT_RET_NOTFOUND_OK(ret); + + /* Setup timestamp usage hints */ + WT_RET(__wt_config_gets(session, cfg, "write_timestamp_usage", &cval)); + if (WT_STRING_MATCH("always", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_ALWAYS); + else if (WT_STRING_MATCH("key_consistent", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_KEY_CONSISTENT); + else if (WT_STRING_MATCH("mixed_mode", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_MIXED_MODE); + else if (WT_STRING_MATCH("never", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_NEVER); + else if (WT_STRING_MATCH("ordered", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_ORDERED); + + return (0); +} + +/* * __wt_conn_dhandle_open -- * Open the current data handle. */ @@ -447,6 +498,7 @@ __wt_conn_dhandle_open(WT_SESSION_IMPL *session, const char *cfg[], uint32_t fla /* Discard any previous configuration, set up the new configuration. */ __conn_dhandle_config_clear(session); WT_ERR(__conn_dhandle_config_set(session)); + WT_ERR(__conn_dhandle_config_parse(session)); switch (dhandle->type) { case WT_DHANDLE_TYPE_BTREE: @@ -465,7 +517,7 @@ __wt_conn_dhandle_open(WT_SESSION_IMPL *session, const char *cfg[], uint32_t fla WT_ERR(__wt_btree_open(session, cfg)); break; case WT_DHANDLE_TYPE_TABLE: - WT_ERR(__wt_schema_open_table(session, cfg)); + WT_ERR(__wt_schema_open_table(session)); break; } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index f3cf48e3c67..35b60b2fe4e 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -277,6 +277,9 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) conn = S2C(session); cache = conn->cache; + /* Mark the session as an eviction thread session. */ + F_SET(session, WT_SESSION_EVICTION); + /* * Cache a history store cursor to avoid deadlock: if an eviction thread thread marks a file * busy and then opens a different file (in this case, the HS file), it can deadlock with a @@ -351,6 +354,9 @@ __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread) */ WT_ASSERT(session, F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_RECOVERING)); + /* Clear the eviction thread session flag. */ + F_CLR(session, WT_SESSION_EVICTION); + __wt_verbose(session, WT_VERB_EVICTSERVER, "%s", "cache eviction thread exiting"); if (0) { @@ -2259,10 +2265,6 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) } } - /* Set a flag to indicate that either eviction server or worker thread is evicting the page. */ - if (F_ISSET(session, WT_SESSION_INTERNAL)) - LF_SET(WT_REC_EVICTION_THREAD); - /* * In case something goes wrong, don't pick the same set of pages every time. * diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 00bc7dbf4c3..cf6bc39bdb7 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -659,7 +659,7 @@ __evict_review(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t evict_flags, bool use_snapshot_for_app_thread = !F_ISSET(session, WT_SESSION_INTERNAL) && !WT_IS_METADATA(session->dhandle) && WT_SESSION_TXN_SHARED(session)->id != WT_TXN_NONE && F_ISSET(session->txn, WT_TXN_HAS_SNAPSHOT); - is_eviction_thread = FLD_ISSET(evict_flags, WT_REC_EVICTION_THREAD); + is_eviction_thread = F_ISSET(session, WT_SESSION_EVICTION); /* Make sure that both conditions above are not true at the same time. */ WT_ASSERT(session, !use_snapshot_for_app_thread || !is_eviction_thread); diff --git a/src/third_party/wiredtiger/src/history/hs_rec.c b/src/third_party/wiredtiger/src/history/hs_rec.c index 5c5a72ea366..b0ea9dbd669 100644 --- a/src/third_party/wiredtiger/src/history/hs_rec.c +++ b/src/third_party/wiredtiger/src/history/hs_rec.c @@ -8,16 +8,6 @@ #include "wt_internal.h" -/* - * WT_HS_TIME_PAIR -- - * A pair containing a timestamp and transaction id. - */ -typedef struct { - wt_timestamp_t ts; - wt_timestamp_t durable_ts; - uint64_t txnid; -} WT_HS_TIME_POINT; - static int __hs_delete_key_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, uint32_t btree_id, const WT_ITEM *key, bool reinsert); static int __hs_fixup_out_of_order_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, @@ -77,8 +67,8 @@ __hs_verbose_cache_stats(WT_SESSION_IMPL *session, WT_BTREE *btree) */ static int __hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, uint64_t btree_id, - const WT_ITEM *key, const uint8_t type, const WT_ITEM *hs_value, - WT_HS_TIME_POINT *start_time_point, WT_HS_TIME_POINT *stop_time_point, uint64_t counter) + const WT_ITEM *key, const uint8_t type, const WT_ITEM *hs_value, WT_TIME_WINDOW *tw, + uint64_t counter) { WT_CURSOR_BTREE *cbt; WT_DECL_RET; @@ -94,20 +84,19 @@ __hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, u * Use WT_CURSOR.set_key and WT_CURSOR.set_value to create key and value items, then use them to * create an update chain for a direct insertion onto the history store page. */ - cursor->set_key(cursor, btree_id, key, start_time_point->ts, counter); - cursor->set_value( - cursor, stop_time_point->durable_ts, start_time_point->durable_ts, (uint64_t)type, hs_value); + cursor->set_key(cursor, btree_id, key, tw->start_ts, counter); + cursor->set_value(cursor, tw->durable_stop_ts, tw->durable_start_ts, (uint64_t)type, hs_value); /* Allocate a tombstone only when there is a valid stop time point. */ - if (stop_time_point->ts != WT_TS_MAX || stop_time_point->txnid != WT_TXN_MAX) { + if (WT_TIME_WINDOW_HAS_STOP(tw)) { /* * Insert a delete record to represent stop time point for the actual record to be inserted. * Set the stop time point as the commit time point of the history store delete record. */ WT_ERR(__wt_upd_alloc_tombstone(session, &hs_upd, NULL)); - hs_upd->start_ts = stop_time_point->ts; - hs_upd->durable_ts = stop_time_point->durable_ts; - hs_upd->txnid = stop_time_point->txnid; + hs_upd->start_ts = tw->stop_ts; + hs_upd->durable_ts = tw->durable_stop_ts; + hs_upd->txnid = tw->stop_txn; } /* @@ -115,9 +104,9 @@ __hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, u * current update start time point as the commit time point to the history store record. */ WT_ERR(__wt_upd_alloc(session, &cursor->value, WT_UPDATE_STANDARD, &upd_local, NULL)); - upd_local->start_ts = start_time_point->ts; - upd_local->durable_ts = start_time_point->durable_ts; - upd_local->txnid = start_time_point->txnid; + upd_local->start_ts = tw->start_ts; + upd_local->durable_ts = tw->durable_start_ts; + upd_local->txnid = tw->start_txn; /* Insert the standard update as next update if there is a tombstone. */ if (hs_upd != NULL) @@ -160,8 +149,7 @@ err: */ static int __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, - const WT_ITEM *key, const uint8_t type, const WT_ITEM *hs_value, - WT_HS_TIME_POINT *start_time_point, WT_HS_TIME_POINT *stop_time_point) + const WT_ITEM *key, const uint8_t type, const WT_ITEM *hs_value, WT_TIME_WINDOW *tw) { #ifdef HAVE_DIAGNOSTIC WT_CURSOR_BTREE *hs_cbt; @@ -221,8 +209,7 @@ __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BT * one can lead to wrong order. */ WT_ERR_NOTFOUND_OK( - __wt_hs_cursor_position(session, cursor, btree->id, key, start_time_point->ts, srch_key), - true); + __wt_hs_cursor_position(session, cursor, btree->id, key, tw->start_ts, srch_key), true); if (ret == 0) { WT_ERR(cursor->get_key(cursor, &hs_btree_id, hs_key, &hs_start_ts, &hs_counter)); /* @@ -231,7 +218,7 @@ __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BT * Verify simple checks first to confirm whether the retrieved update same or not before * performing the expensive key comparison. */ - if (hs_btree_id == btree->id && start_time_point->ts == hs_start_ts) { + if (hs_btree_id == btree->id && tw->start_ts == hs_start_ts) { WT_ERR(__wt_compare(session, NULL, hs_key, key, &cmp)); #ifdef HAVE_DIAGNOSTIC @@ -241,9 +228,9 @@ __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BT WT_ERR(__wt_compare(session, NULL, existing_val, hs_value, &cmp)); if (cmp == 0) WT_ASSERT(session, - start_time_point->txnid == WT_TXN_NONE || - start_time_point->txnid != hs_cbt->upd_value->tw.start_txn || - start_time_point->ts != hs_cbt->upd_value->tw.start_ts); + tw->start_txn == WT_TXN_NONE || + tw->start_txn != hs_cbt->upd_value->tw.start_txn || + tw->start_ts != hs_cbt->upd_value->tw.start_ts); counter = hs_counter + 1; } #else @@ -257,11 +244,11 @@ __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BT * If we're inserting a non-zero timestamp, look ahead for any higher timestamps. If we find * updates, we should remove them and reinsert them at the current timestamp. */ - if (start_time_point->ts != WT_TS_NONE) { + if (tw->start_ts != WT_TS_NONE) { WT_ERR_NOTFOUND_OK(__wt_hs_cursor_next(session, cursor), true); if (ret == 0) WT_ERR(__hs_fixup_out_of_order_from_pos( - session, cursor, btree, key, start_time_point->ts, &counter, srch_key)); + session, cursor, btree, key, tw->start_ts, &counter, srch_key)); } #ifdef HAVE_DIAGNOSTIC @@ -271,16 +258,15 @@ __hs_insert_record_with_btree(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BT */ if (F_ISSET(cursor, WT_CURSTD_KEY_SET)) { WT_ERR(cursor->get_key(cursor, &hs_btree_id, hs_key, &hs_start_ts, &hs_counter)); - if (hs_btree_id == btree->id && start_time_point->ts == hs_start_ts && - hs_counter == counter) { + if (hs_btree_id == btree->id && tw->start_ts == hs_start_ts && hs_counter == counter) { WT_ERR(__wt_compare(session, NULL, hs_key, key, &cmp)); WT_ASSERT(session, cmp != 0); } } #endif /* The tree structure can change while we try to insert the mod list, retry if that happens. */ - while ((ret = __hs_insert_record_with_btree_int(session, cursor, btree->id, key, type, hs_value, - start_time_point, stop_time_point, counter)) == WT_RESTART) { + while ((ret = __hs_insert_record_with_btree_int( + session, cursor, btree->id, key, type, hs_value, tw, counter)) == WT_RESTART) { WT_STAT_CONN_INCR(session, cache_hs_insert_restart); WT_STAT_DATA_INCR(session, cache_hs_insert_restart); } @@ -302,16 +288,14 @@ err: */ static int __hs_insert_record(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_BTREE *btree, const WT_ITEM *key, - const uint8_t type, const WT_ITEM *hs_value, WT_HS_TIME_POINT *start_time_point, - WT_HS_TIME_POINT *stop_time_point) + const uint8_t type, const WT_ITEM *hs_value, WT_TIME_WINDOW *tw) { WT_CURSOR_BTREE *cbt; WT_DECL_RET; cbt = (WT_CURSOR_BTREE *)cursor; WT_WITH_BTREE(session, CUR2BT(cbt), - ret = __hs_insert_record_with_btree( - session, cursor, btree, key, type, hs_value, start_time_point, stop_time_point)); + ret = __hs_insert_record_with_btree(session, cursor, btree, key, type, hs_value, tw)); return (ret); } @@ -373,7 +357,7 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) WT_SAVE_UPD *list; WT_UPDATE *first_globally_visible_upd, *first_non_ts_upd; WT_UPDATE *non_aborted_upd, *oldest_upd, *prev_upd, *tombstone, *upd; - WT_HS_TIME_POINT start_time_point, stop_time_point; + WT_TIME_WINDOW tw; wt_off_t hs_size; wt_timestamp_t min_insert_ts; uint64_t insert_cnt, max_hs_size; @@ -387,6 +371,7 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) cursor = session->hs_cursor; prev_upd = NULL; insert_cnt = 0; + WT_TIME_WINDOW_INIT(&tw); __wt_modify_vector_init(session, &modifies); if (!btree->hs_entries) @@ -600,9 +585,9 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) upd = prev_upd) { WT_ASSERT(session, upd->type == WT_UPDATE_STANDARD || upd->type == WT_UPDATE_MODIFY); - start_time_point.durable_ts = upd->durable_ts; - start_time_point.ts = upd->start_ts; - start_time_point.txnid = upd->txnid; + tw.durable_start_ts = upd->durable_ts; + tw.start_ts = upd->start_ts; + tw.start_txn = upd->txnid; tombstone = NULL; __wt_modify_vector_peek(&modifies, &prev_upd); @@ -615,8 +600,8 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) WT_ASSERT(session, list->onpage_upd->txnid == prev_upd->txnid && list->onpage_upd->start_ts == prev_upd->start_ts); - stop_time_point.durable_ts = stop_time_point.ts = WT_TS_MAX; - stop_time_point.txnid = WT_TXN_MAX; + tw.durable_stop_ts = tw.stop_ts = WT_TS_MAX; + tw.stop_txn = WT_TXN_MAX; } else { /* * Set the stop timestamp from durable timestamp instead of commit timestamp. The @@ -624,9 +609,9 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) * timestamp is globally visible. i.e. durable timestamp of data store version. */ WT_ASSERT(session, prev_upd->start_ts <= prev_upd->durable_ts); - stop_time_point.durable_ts = prev_upd->durable_ts; - stop_time_point.ts = prev_upd->start_ts; - stop_time_point.txnid = prev_upd->txnid; + tw.durable_stop_ts = prev_upd->durable_ts; + tw.stop_ts = prev_upd->start_ts; + tw.stop_txn = prev_upd->txnid; if (prev_upd->type == WT_UPDATE_TOMBSTONE) tombstone = prev_upd; @@ -656,8 +641,8 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) * rules. As such older readers should still be able to see it. */ if (F_ISSET(upd, WT_UPDATE_BEHIND_MIXED_MODE)) { - start_time_point.ts = start_time_point.durable_ts = WT_TS_NONE; - stop_time_point.ts = stop_time_point.durable_ts = WT_TS_NONE; + tw.start_ts = tw.durable_start_ts = WT_TS_NONE; + tw.stop_ts = tw.durable_stop_ts = WT_TS_NONE; } /* @@ -669,11 +654,11 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) * FIXME-WT-6443: We should be able to replace this with an assertion. */ if (!F_ISSET(upd, WT_UPDATE_BEHIND_MIXED_MODE) && - (stop_time_point.ts < upd->start_ts || - (stop_time_point.ts == upd->start_ts && stop_time_point.txnid <= upd->txnid))) { + (tw.stop_ts < upd->start_ts || + (tw.stop_ts == upd->start_ts && tw.stop_txn <= upd->txnid))) { __wt_verbose(session, WT_VERB_TIMESTAMP, "Warning: fixing out-of-order timestamps %s earlier than previous update %s", - __wt_timestamp_to_string(stop_time_point.ts, ts_string[0]), + __wt_timestamp_to_string(tw.stop_ts, ts_string[0]), __wt_timestamp_to_string(upd->start_ts, ts_string[1])); continue; } @@ -712,12 +697,12 @@ __wt_hs_insert_updates(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi) __wt_calc_modify(session, prev_full_value, full_value, prev_full_value->size / 10, entries, &nentries) == 0) { WT_ERR(__wt_modify_pack(cursor, entries, nentries, &modify_value)); - WT_ERR(__hs_insert_record(session, cursor, btree, key, WT_UPDATE_MODIFY, - modify_value, &start_time_point, &stop_time_point)); + WT_ERR(__hs_insert_record( + session, cursor, btree, key, WT_UPDATE_MODIFY, modify_value, &tw)); __wt_scr_free(session, &modify_value); } else - WT_ERR(__hs_insert_record(session, cursor, btree, key, WT_UPDATE_STANDARD, - full_value, &start_time_point, &stop_time_point)); + WT_ERR(__hs_insert_record( + session, cursor, btree, key, WT_UPDATE_STANDARD, full_value, &tw)); /* Flag the update as now in the history store. */ F_SET(upd, WT_UPDATE_HS); @@ -864,11 +849,10 @@ __hs_fixup_out_of_order_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_CURSOR *insert_cursor; WT_CURSOR_BTREE *hs_cbt; WT_DECL_RET; - WT_HS_TIME_POINT start_time_point, stop_time_point; WT_ITEM hs_key, hs_value; WT_TIME_WINDOW tw; WT_UPDATE *tombstone; - wt_timestamp_t hs_ts; + wt_timestamp_t hs_ts, hs_start_durable_ts, hs_stop_durable_ts; uint64_t hs_counter, hs_upd_type; uint32_t hs_btree_id; int cmp; @@ -879,6 +863,7 @@ __hs_fixup_out_of_order_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, hs_cbt = (WT_CURSOR_BTREE *)hs_cursor; WT_CLEAR(hs_key); WT_CLEAR(hs_value); + WT_TIME_WINDOW_INIT(&tw); tombstone = NULL; /* The session should be pointing at the history store btree. */ @@ -988,25 +973,24 @@ __hs_fixup_out_of_order_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, __wt_timestamp_to_string(hs_cbt->upd_value->tw.durable_stop_ts, ts_string[3]), __wt_timestamp_to_string(ts, ts_string[4])); - start_time_point.ts = start_time_point.durable_ts = ts; - start_time_point.txnid = hs_cbt->upd_value->tw.start_txn; + tw.start_ts = tw.durable_start_ts = ts; + tw.start_txn = hs_cbt->upd_value->tw.start_txn; /* * We're going to be inserting something immediately after with the same timestamp. Either * another moved update OR the update itself that triggered the correction. In either case, * we should preserve the stop transaction id. */ - stop_time_point.ts = stop_time_point.durable_ts = ts; - stop_time_point.txnid = hs_cbt->upd_value->tw.stop_txn; + tw.stop_ts = tw.durable_stop_ts = ts; + tw.stop_txn = hs_cbt->upd_value->tw.stop_txn; /* Extract the underlying value for reinsertion. */ WT_ERR(hs_cursor->get_value( - hs_cursor, &tw.durable_stop_ts, &tw.durable_start_ts, &hs_upd_type, &hs_value)); + hs_cursor, &hs_stop_durable_ts, &hs_start_durable_ts, &hs_upd_type, &hs_value)); /* Reinsert entry with earlier timestamp. */ while ((ret = __hs_insert_record_with_btree_int(session, insert_cursor, btree->id, key, - (uint8_t)hs_upd_type, &hs_value, &start_time_point, &stop_time_point, - *counter)) == WT_RESTART) + (uint8_t)hs_upd_type, &hs_value, &tw, *counter)) == WT_RESTART) ; WT_ERR(ret); ++(*counter); @@ -1047,8 +1031,8 @@ __hs_delete_key_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, uint32_ WT_CURSOR *insert_cursor; WT_CURSOR_BTREE *hs_cbt; WT_DECL_RET; - WT_HS_TIME_POINT start_time_point, stop_time_point; WT_ITEM hs_key, hs_value; + WT_TIME_WINDOW tw; WT_UPDATE *upd; wt_timestamp_t durable_timestamp, hs_start_ts, hs_stop_durable_ts; uint64_t hs_counter, hs_insert_counter, hs_upd_type; @@ -1060,6 +1044,7 @@ __hs_delete_key_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, uint32_ hs_insert_counter = 0; WT_CLEAR(hs_key); WT_CLEAR(hs_value); + WT_TIME_WINDOW_INIT(&tw); upd = NULL; insert_cursor = NULL; @@ -1128,16 +1113,16 @@ __hs_delete_key_from_pos(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, uint32_ WT_ERR(hs_cursor->get_value( hs_cursor, &hs_stop_durable_ts, &durable_timestamp, &hs_upd_type, &hs_value)); - start_time_point.ts = start_time_point.durable_ts = WT_TS_NONE; - start_time_point.txnid = hs_cbt->upd_value->tw.start_txn; + tw.start_ts = tw.durable_start_ts = WT_TS_NONE; + tw.start_txn = hs_cbt->upd_value->tw.start_txn; - stop_time_point.ts = stop_time_point.durable_ts = WT_TS_NONE; - stop_time_point.txnid = hs_cbt->upd_value->tw.stop_txn; + tw.stop_ts = tw.durable_stop_ts = WT_TS_NONE; + tw.stop_txn = hs_cbt->upd_value->tw.stop_txn; /* Reinsert entry with zero timestamp. */ - while ((ret = __hs_insert_record_with_btree_int(session, insert_cursor, btree_id, - &hs_key, (uint8_t)hs_upd_type, &hs_value, &start_time_point, &stop_time_point, - hs_insert_counter)) == WT_RESTART) + while ( + (ret = __hs_insert_record_with_btree_int(session, insert_cursor, btree_id, &hs_key, + (uint8_t)hs_upd_type, &hs_value, &tw, hs_insert_counter)) == WT_RESTART) ; hs_insert_counter++; WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 3cc12d78e74..2647d9ad7de 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -30,12 +30,11 @@ #define WT_REC_CHECKPOINT 0x004u #define WT_REC_CLEAN_AFTER_REC 0x008u #define WT_REC_EVICT 0x010u -#define WT_REC_EVICTION_THREAD 0x020u -#define WT_REC_HS 0x040u -#define WT_REC_IN_MEMORY 0x080u -#define WT_REC_SCRUB 0x100u -#define WT_REC_VISIBILITY_ERR 0x200u -#define WT_REC_VISIBLE_ALL 0x400u +#define WT_REC_HS 0x020u +#define WT_REC_IN_MEMORY 0x040u +#define WT_REC_SCRUB 0x080u +#define WT_REC_VISIBILITY_ERR 0x100u +#define WT_REC_VISIBLE_ALL 0x200u /* AUTOMATIC FLAG VALUE GENERATION STOP */ /* @@ -1054,6 +1053,9 @@ struct __wt_update { wt_timestamp_t durable_ts; /* timestamps */ wt_timestamp_t start_ts; +#ifdef HAVE_DIAGNOSTIC + wt_timestamp_t prev_durable_ts; +#endif WT_UPDATE *next; /* forward-linked list */ @@ -1099,7 +1101,11 @@ struct __wt_update { * WT_UPDATE_SIZE is the expected structure size excluding the payload data -- we verify the build * to ensure the compiler hasn't inserted padding. */ +#ifdef HAVE_DIAGNOSTIC +#define WT_UPDATE_SIZE 47 +#else #define WT_UPDATE_SIZE 39 +#endif /* * The memory size of an update: include some padding because this is such a common case that diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 4af6ea90f67..85ddcff102d 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -100,18 +100,6 @@ struct __wt_btree { uint32_t maxmempage_image; /* In-memory page image max size */ uint64_t splitmempage; /* In-memory split trigger size */ -/* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_ASSERT_COMMIT_TS_ALWAYS 0x01u -#define WT_ASSERT_COMMIT_TS_KEYS 0x02u -#define WT_ASSERT_COMMIT_TS_NEVER 0x04u -#define WT_ASSERT_DURABLE_TS_ALWAYS 0x08u -#define WT_ASSERT_DURABLE_TS_KEYS 0x10u -#define WT_ASSERT_DURABLE_TS_NEVER 0x20u -#define WT_ASSERT_READ_TS_ALWAYS 0x40u -#define WT_ASSERT_READ_TS_NEVER 0x80u - /* AUTOMATIC FLAG VALUE GENERATION STOP */ - uint32_t assert_flags; /* Debugging assertion information */ - void *huffman_value; /* Value huffman encoding */ enum { diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index ca28d17d4c5..d52fe680d81 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -102,6 +102,15 @@ struct __wt_named_extractor { }; /* + * WT_NAME_FLAG -- + * Simple structure for name and flag configuration searches + */ +struct __wt_name_flag { + const char *name; + uint64_t flag; +}; + +/* * WT_CONN_CHECK_PANIC -- * Check if we've panicked and return the appropriate error. */ @@ -515,16 +524,17 @@ struct __wt_connection_impl { #define WT_TIMING_STRESS_BACKUP_RENAME 0x0002u #define WT_TIMING_STRESS_CHECKPOINT_SLOW 0x0004u #define WT_TIMING_STRESS_HS_CHECKPOINT_DELAY 0x0008u -#define WT_TIMING_STRESS_HS_SWEEP 0x0010u -#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x0020u -#define WT_TIMING_STRESS_SPLIT_1 0x0040u -#define WT_TIMING_STRESS_SPLIT_2 0x0080u -#define WT_TIMING_STRESS_SPLIT_3 0x0100u -#define WT_TIMING_STRESS_SPLIT_4 0x0200u -#define WT_TIMING_STRESS_SPLIT_5 0x0400u -#define WT_TIMING_STRESS_SPLIT_6 0x0800u -#define WT_TIMING_STRESS_SPLIT_7 0x1000u -#define WT_TIMING_STRESS_SPLIT_8 0x2000u +#define WT_TIMING_STRESS_HS_SEARCH 0x0010u +#define WT_TIMING_STRESS_HS_SWEEP 0x0020u +#define WT_TIMING_STRESS_PREPARE_CHECKPOINT_DELAY 0x0040u +#define WT_TIMING_STRESS_SPLIT_1 0x0080u +#define WT_TIMING_STRESS_SPLIT_2 0x0100u +#define WT_TIMING_STRESS_SPLIT_3 0x0200u +#define WT_TIMING_STRESS_SPLIT_4 0x0400u +#define WT_TIMING_STRESS_SPLIT_5 0x0800u +#define WT_TIMING_STRESS_SPLIT_6 0x1000u +#define WT_TIMING_STRESS_SPLIT_7 0x2000u +#define WT_TIMING_STRESS_SPLIT_8 0x4000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint64_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 49e9e756fe6..2ee23b71701 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -112,4 +112,17 @@ struct __wt_data_handle { #define WT_DHANDLE_OPEN 0x80u /* Handle is open */ /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_DHANDLE_ASSERT_TS_READ_ALWAYS 0x001u /* Assert read always checking. */ +#define WT_DHANDLE_ASSERT_TS_READ_NEVER 0x002u /* Assert read never checking. */ +#define WT_DHANDLE_ASSERT_TS_WRITE 0x004u /* Assert write checking. */ +#define WT_DHANDLE_TS_ALWAYS 0x008u /* Handle using always checking. */ +#define WT_DHANDLE_TS_KEY_CONSISTENT 0x010u /* Handle using key consistency checking. */ +#define WT_DHANDLE_TS_MIXED_MODE 0x020u /* Handle using mixed mode timestamps checking. */ +#define WT_DHANDLE_TS_NEVER 0x040u /* Handle never using timestamps checking. */ +#define WT_DHANDLE_TS_ORDERED 0x080u /* Handle using ordered timestamps checking. */ +#define WT_DHANDLE_VERB_TS_WRITE 0x100u /* Handle verbose logging for timestamps usage. */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t ts_flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index d902d72ff01..1c646d6622c 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1289,7 +1289,7 @@ extern int __wt_schema_open_index(WT_SESSION_IMPL *session, WT_TABLE *table, con size_t len, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_indices(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) +extern int __wt_schema_open_table(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_project_in(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -2048,7 +2048,7 @@ static inline int __wt_txn_read_upd_list(WT_SESSION_IMPL *session, WT_CURSOR_BTR static inline int __wt_txn_search_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + WT_UPDATE *upd, wt_timestamp_t *prev_tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, u_int modify_type, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp, diff --git a/src/third_party/wiredtiger/src/include/serial_inline.h b/src/third_party/wiredtiger/src/include/serial_inline.h index 11e49c9081a..a469b0db0f6 100644 --- a/src/third_party/wiredtiger/src/include/serial_inline.h +++ b/src/third_party/wiredtiger/src/include/serial_inline.h @@ -222,12 +222,13 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page { WT_DECL_RET; WT_UPDATE *obsolete, *upd; - wt_timestamp_t obsolete_timestamp; + wt_timestamp_t obsolete_timestamp, prev_upd_ts; uint64_t txn; /* Clear references to memory we now own and must free on error. */ upd = *updp; *updp = NULL; + prev_upd_ts = WT_TS_NONE; /* * All structure setup must be flushed before the structure is entered into the list. We need a @@ -237,12 +238,15 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page * Check if our update is still permitted. */ while (!__wt_atomic_cas_ptr(srch_upd, upd->next, upd)) { - if ((ret = __wt_txn_update_check(session, cbt, upd->next = *srch_upd)) != 0) { + if ((ret = __wt_txn_update_check(session, cbt, upd->next = *srch_upd, &prev_upd_ts)) != 0) { /* Free unused memory on error. */ __wt_free(session, upd); return (ret); } } +#ifdef HAVE_DIAGNOSTIC + upd->prev_durable_ts = prev_upd_ts; +#endif /* * Increment in-memory footprint after swapping the update into place. Safe because the diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 0186b98f4cc..9f11361aaf2 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -169,33 +169,34 @@ struct __wt_session_impl { #define WT_SESSION_BACKUP_DUP 0x00000002u #define WT_SESSION_CACHE_CURSORS 0x00000004u #define WT_SESSION_CAN_WAIT 0x00000008u -#define WT_SESSION_IGNORE_CACHE_SIZE 0x00000010u -#define WT_SESSION_IMPORT 0x00000020u -#define WT_SESSION_IMPORT_REPAIR 0x00000040u -#define WT_SESSION_INSTANTIATE_PREPARE 0x00000080u -#define WT_SESSION_INTERNAL 0x00000100u -#define WT_SESSION_LOCKED_CHECKPOINT 0x00000200u -#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000400u -#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00000800u -#define WT_SESSION_LOCKED_HOTBACKUP_READ 0x00001000u -#define WT_SESSION_LOCKED_HOTBACKUP_WRITE 0x00002000u -#define WT_SESSION_LOCKED_METADATA 0x00004000u -#define WT_SESSION_LOCKED_PASS 0x00008000u -#define WT_SESSION_LOCKED_SCHEMA 0x00010000u -#define WT_SESSION_LOCKED_SLOT 0x00020000u -#define WT_SESSION_LOCKED_TABLE_READ 0x00040000u -#define WT_SESSION_LOCKED_TABLE_WRITE 0x00080000u -#define WT_SESSION_LOCKED_TURTLE 0x00100000u -#define WT_SESSION_LOGGING_INMEM 0x00200000u -#define WT_SESSION_NO_DATA_HANDLES 0x00400000u -#define WT_SESSION_NO_LOGGING 0x00800000u -#define WT_SESSION_NO_RECONCILE 0x01000000u -#define WT_SESSION_NO_SCHEMA_LOCK 0x02000000u -#define WT_SESSION_QUIET_CORRUPT_FILE 0x04000000u -#define WT_SESSION_READ_WONT_NEED 0x08000000u -#define WT_SESSION_RESOLVING_TXN 0x10000000u -#define WT_SESSION_ROLLBACK_TO_STABLE 0x20000000u -#define WT_SESSION_SCHEMA_TXN 0x40000000u +#define WT_SESSION_EVICTION 0x00000010u +#define WT_SESSION_IGNORE_CACHE_SIZE 0x00000020u +#define WT_SESSION_IMPORT 0x00000040u +#define WT_SESSION_IMPORT_REPAIR 0x00000080u +#define WT_SESSION_INSTANTIATE_PREPARE 0x00000100u +#define WT_SESSION_INTERNAL 0x00000200u +#define WT_SESSION_LOCKED_CHECKPOINT 0x00000400u +#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000800u +#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00001000u +#define WT_SESSION_LOCKED_HOTBACKUP_READ 0x00002000u +#define WT_SESSION_LOCKED_HOTBACKUP_WRITE 0x00004000u +#define WT_SESSION_LOCKED_METADATA 0x00008000u +#define WT_SESSION_LOCKED_PASS 0x00010000u +#define WT_SESSION_LOCKED_SCHEMA 0x00020000u +#define WT_SESSION_LOCKED_SLOT 0x00040000u +#define WT_SESSION_LOCKED_TABLE_READ 0x00080000u +#define WT_SESSION_LOCKED_TABLE_WRITE 0x00100000u +#define WT_SESSION_LOCKED_TURTLE 0x00200000u +#define WT_SESSION_LOGGING_INMEM 0x00400000u +#define WT_SESSION_NO_DATA_HANDLES 0x00800000u +#define WT_SESSION_NO_LOGGING 0x01000000u +#define WT_SESSION_NO_RECONCILE 0x02000000u +#define WT_SESSION_NO_SCHEMA_LOCK 0x04000000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x08000000u +#define WT_SESSION_READ_WONT_NEED 0x10000000u +#define WT_SESSION_RESOLVING_TXN 0x20000000u +#define WT_SESSION_ROLLBACK_TO_STABLE 0x40000000u +#define WT_SESSION_SCHEMA_TXN 0x80000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index c3b7f37d8fe..ddfcf99a52d 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -340,16 +340,16 @@ struct __wt_txn { #define WT_TXN_SHARED_TS_DURABLE 0x000800u #define WT_TXN_SHARED_TS_READ 0x001000u #define WT_TXN_SYNC_SET 0x002000u -#define WT_TXN_TS_COMMIT_ALWAYS 0x004000u -#define WT_TXN_TS_COMMIT_KEYS 0x008000u -#define WT_TXN_TS_COMMIT_NEVER 0x010000u -#define WT_TXN_TS_DURABLE_ALWAYS 0x020000u -#define WT_TXN_TS_DURABLE_KEYS 0x040000u -#define WT_TXN_TS_DURABLE_NEVER 0x080000u -#define WT_TXN_TS_READ_BEFORE_OLDEST 0x100000u -#define WT_TXN_TS_ROUND_PREPARED 0x200000u -#define WT_TXN_TS_ROUND_READ 0x400000u -#define WT_TXN_UPDATE 0x800000u +#define WT_TXN_TS_READ_BEFORE_OLDEST 0x004000u +#define WT_TXN_TS_ROUND_PREPARED 0x008000u +#define WT_TXN_TS_ROUND_READ 0x010000u +#define WT_TXN_TS_WRITE_ALWAYS 0x020000u +#define WT_TXN_TS_WRITE_KEY_CONSISTENT 0x040000u +#define WT_TXN_TS_WRITE_MIXED_MODE 0x080000u +#define WT_TXN_TS_WRITE_NEVER 0x100000u +#define WT_TXN_TS_WRITE_ORDERED 0x200000u +#define WT_TXN_UPDATE 0x400000u +#define WT_TXN_VERB_TS_WRITE 0x800000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index 629906d8ef1..0433a252d63 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -71,24 +71,32 @@ static inline void __wt_txn_timestamp_flags(WT_SESSION_IMPL *session) { WT_BTREE *btree; + WT_DATA_HANDLE *dhandle; - if (session->dhandle == NULL) + dhandle = session->dhandle; + if (dhandle == NULL) return; btree = S2BT(session); if (btree == NULL) return; - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_COMMIT_TS_ALWAYS)) - F_SET(session->txn, WT_TXN_TS_COMMIT_ALWAYS); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_COMMIT_TS_KEYS)) - F_SET(session->txn, WT_TXN_TS_COMMIT_KEYS); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_COMMIT_TS_NEVER)) - F_SET(session->txn, WT_TXN_TS_COMMIT_NEVER); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_DURABLE_TS_ALWAYS)) - F_SET(session->txn, WT_TXN_TS_DURABLE_ALWAYS); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_DURABLE_TS_KEYS)) - F_SET(session->txn, WT_TXN_TS_DURABLE_KEYS); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_DURABLE_TS_NEVER)) - F_SET(session->txn, WT_TXN_TS_DURABLE_NEVER); + + if (!FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_WRITE)) + return; + + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_ALWAYS)) + F_SET(session->txn, WT_TXN_TS_WRITE_ALWAYS); + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_KEY_CONSISTENT)) + F_SET(session->txn, WT_TXN_TS_WRITE_KEY_CONSISTENT); + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_MIXED_MODE)) + F_SET(session->txn, WT_TXN_TS_WRITE_MIXED_MODE); + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_NEVER)) + F_SET(session->txn, WT_TXN_TS_WRITE_NEVER); + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_ORDERED)) + F_SET(session->txn, WT_TXN_TS_WRITE_ORDERED); + + /* Remember if any type of verbose tracking is encountered by the transaction. */ + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_VERB_TS_WRITE)) + F_SET(session->txn, WT_TXN_VERB_TS_WRITE); } /* @@ -836,7 +844,7 @@ __wt_txn_read_upd_list( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, WT_UPDATE **prepare_updp) { WT_VISIBLE_TYPE upd_visible; - uint8_t type; + uint8_t prepare_state, type; if (prepare_updp != NULL) *prepare_updp = NULL; @@ -848,6 +856,7 @@ __wt_txn_read_upd_list( if (type == WT_UPDATE_RESERVE) continue; + WT_ORDERED_READ(prepare_state, upd->prepare_state); /* * If the cursor is configured to ignore tombstones, copy the timestamps from the tombstones * to the stop time window of the update value being returned to the caller. Caller can @@ -860,8 +869,8 @@ __wt_txn_read_upd_list( cbt->upd_value->tw.durable_stop_ts = upd->durable_ts; cbt->upd_value->tw.stop_ts = upd->start_ts; cbt->upd_value->tw.stop_txn = upd->txnid; - cbt->upd_value->tw.prepare = upd->prepare_state == WT_PREPARE_INPROGRESS || - upd->prepare_state == WT_PREPARE_LOCKED; + cbt->upd_value->tw.prepare = + prepare_state == WT_PREPARE_INPROGRESS || prepare_state == WT_PREPARE_LOCKED; continue; } @@ -870,18 +879,20 @@ __wt_txn_read_upd_list( if (upd_visible == WT_VISIBLE_TRUE) break; + /* + * Save the prepared update to help us detect if we race with prepared commit or rollback + * irrespective of update visibility. + */ + if ((prepare_state == WT_PREPARE_INPROGRESS || prepare_state == WT_PREPARE_LOCKED) && + prepare_updp != NULL && *prepare_updp == NULL && + F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS)) + *prepare_updp = upd; + if (upd_visible == WT_VISIBLE_PREPARE) { /* Ignore the prepared update, if transaction configuration says so. */ - if (F_ISSET(session->txn, WT_TXN_IGNORE_PREPARE)) { - /* - * Save the prepared update to help us detect if we race with prepared commit or - * rollback. - */ - if (prepare_updp != NULL && *prepare_updp == NULL && - F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS)) - *prepare_updp = upd; + if (F_ISSET(session->txn, WT_TXN_IGNORE_PREPARE)) continue; - } + return (WT_PREPARE_CONFLICT); } } @@ -992,9 +1003,11 @@ retry: } /* If there's no visible update in the update chain or ondisk, check the history store file. */ - if (F_ISSET(S2C(session), WT_CONN_HS_OPEN) && !F_ISSET(S2BT(session), WT_BTREE_HS)) + if (F_ISSET(S2C(session), WT_CONN_HS_OPEN) && !F_ISSET(S2BT(session), WT_BTREE_HS)) { + __wt_timing_stress(session, WT_TIMING_STRESS_HS_SEARCH); WT_RET(__wt_hs_find_upd(session, key, cbt->iface.value_format, recno, cbt->upd_value, false, &cbt->upd_value->buf, &tw)); + } /* * Retry if we race with prepared commit or rollback. If we race with prepared rollback, the @@ -1209,10 +1222,10 @@ __wt_txn_search_check(WT_SESSION_IMPL *session) * Same if it should never have a read timestamp. */ if (!F_ISSET(S2C(session), WT_CONN_RECOVERING) && - FLD_ISSET(btree->assert_flags, WT_ASSERT_READ_TS_ALWAYS) && + FLD_ISSET(btree->dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_ALWAYS) && !F_ISSET(txn, WT_TXN_SHARED_TS_READ)) WT_RET_MSG(session, EINVAL, "read_timestamp required and none set on this transaction"); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_READ_TS_NEVER) && + if (FLD_ISSET(btree->dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_NEVER) && F_ISSET(txn, WT_TXN_SHARED_TS_READ)) WT_RET_MSG( session, EINVAL, "no read_timestamp required and timestamp set on this transaction"); @@ -1224,7 +1237,8 @@ __wt_txn_search_check(WT_SESSION_IMPL *session) * Check if the current transaction can update an item. */ static inline int -__wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) +__wt_txn_update_check( + WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, wt_timestamp_t *prev_tsp) { WT_DECL_RET; WT_TIME_WINDOW tw; @@ -1277,6 +1291,16 @@ __wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE ret = __wt_txn_rollback_required(session, "conflict between concurrent operations"); } + if (prev_tsp != NULL && upd != NULL) { + /* + * The durable timestamp must be greater than or equal to the commit timestamp unless it is + * an in-progress prepared update. + * + * FIXME-WT-7020: We should be able to assert this but we're seeing some fallout in format. + * We should investigate why and assert the above statement. + */ + *prev_tsp = upd->durable_ts; + } if (ignore_prepare_set) F_SET(txn, WT_TXN_IGNORE_PREPARE); return (ret); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index f44b4e188c2..249e6629680 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1002,6 +1002,24 @@ struct __wt_session { * object blocks from the system buffer cache after that many bytes from this object are * read or written into the buffer cache., an integer greater than or equal to 0; default \c * 0.} + * @config{verbose, enable messages for various events. Options are given as a list\, such + * as <code>"verbose=[write_timestamp]"</code>., a list\, with values chosen from the + * following options: \c "write_timestamp"; default \c [].} + * @config{write_timestamp_usage, describe how timestamps are expected to be used on + * modifications to the table. This option should be used in conjunction with the + * corresponding \c write_timestamp configuration under the \c assert and \c verbose options + * to provide logging and assertions for incorrect timestamp usage. The choices are \c + * always which ensures a timestamp is used for every operation on a table\, \c + * key_consistent to ensure that once timestamps are used for a key\, they are always used\, + * \c ordered is like \c key_consistent except it also enforces that subsequent updates to + * each key must use increasing timestamps\, \c mixed_mode is like \c ordered except that + * updates with no timestamp are allowed and have the effect of resetting the chain of + * updates once the transaction ID based snapshot is no longer relevant\, \c never enforces + * that timestamps are never used for a table and \c none does not enforce any expectation + * on timestamp usage meaning that no log message or assertions will be produced regardless + * of the corresponding \c assert and \c verbose settings., a string\, chosen from the + * following options: \c "always"\, \c "key_consistent"\, \c "mixed_mode"\, \c "never"\, \c + * "none"\, \c "ordered"; default \c none.} * @configend * @errors */ @@ -1225,6 +1243,24 @@ struct __wt_session { * applications use a WT_ITEM structure to manipulate raw byte arrays. Value items of type * 't' are bitfields\, and when configured with record number type keys\, will be stored * using a fixed-length store., a format string; default \c u.} + * @config{verbose, enable messages for various events. Options are given as a list\, such + * as <code>"verbose=[write_timestamp]"</code>., a list\, with values chosen from the + * following options: \c "write_timestamp"; default \c [].} + * @config{write_timestamp_usage, describe how timestamps are expected to be used on + * modifications to the table. This option should be used in conjunction with the + * corresponding \c write_timestamp configuration under the \c assert and \c verbose options + * to provide logging and assertions for incorrect timestamp usage. The choices are \c + * always which ensures a timestamp is used for every operation on a table\, \c + * key_consistent to ensure that once timestamps are used for a key\, they are always used\, + * \c ordered is like \c key_consistent except it also enforces that subsequent updates to + * each key must use increasing timestamps\, \c mixed_mode is like \c ordered except that + * updates with no timestamp are allowed and have the effect of resetting the chain of + * updates once the transaction ID based snapshot is no longer relevant\, \c never enforces + * that timestamps are never used for a table and \c none does not enforce any expectation + * on timestamp usage meaning that no log message or assertions will be produced regardless + * of the corresponding \c assert and \c verbose settings., a string\, chosen from the + * following options: \c "always"\, \c "key_consistent"\, \c "mixed_mode"\, \c "never"\, \c + * "none"\, \c "ordered"; default \c none.} * @configend * @errors */ @@ -2139,7 +2175,7 @@ struct __wt_connection { * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "reconcile"\, * \c "recovery"\, \c "recovery_progress"\, \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c * "split"\, \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c - * "verify"\, \c "version"\, \c "write"; default empty.} + * "verify"\, \c "version"\, \c "write"; default \c [].} * @configend * @errors */ @@ -2856,7 +2892,7 @@ struct __wt_connection { * "history_store"\, \c "history_store_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c * "mutex"\, \c "overflow"\, \c "read"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, * \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, - * \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; default empty.} + * \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; default \c [].} * @config{verify_metadata, open connection and verify any WiredTiger metadata. This API allows * verification and detection of corruption in WiredTiger metadata., a boolean flag; default \c * false.} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index a64fb5acdfe..d3aa5905d4c 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -241,6 +241,8 @@ struct __wt_multi; typedef struct __wt_multi WT_MULTI; struct __wt_myslot; typedef struct __wt_myslot WT_MYSLOT; +struct __wt_name_flag; +typedef struct __wt_name_flag WT_NAME_FLAG; struct __wt_named_collator; typedef struct __wt_named_collator WT_NAMED_COLLATOR; struct __wt_named_compressor; diff --git a/src/third_party/wiredtiger/src/schema/schema_open.c b/src/third_party/wiredtiger/src/schema/schema_open.c index 79a07f95912..3af4f12606a 100644 --- a/src/third_party/wiredtiger/src/schema/schema_open.c +++ b/src/third_party/wiredtiger/src/schema/schema_open.c @@ -393,7 +393,7 @@ __wt_schema_open_indices(WT_SESSION_IMPL *session, WT_TABLE *table) * Open the data handle for a table (internal version). */ static int -__schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) +__schema_open_table(WT_SESSION_IMPL *session) { WT_CONFIG cparser; WT_CONFIG_ITEM ckey, cval; @@ -407,7 +407,6 @@ __schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) tablename = table->iface.name; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); - WT_UNUSED(cfg); WT_RET(__wt_config_gets(session, table_cfg, "columns", &cval)); WT_RET(__wt_config_gets(session, table_cfg, "key_format", &cval)); @@ -551,13 +550,12 @@ err: * Open a named table. */ int -__wt_schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_schema_open_table(WT_SESSION_IMPL *session) { WT_DECL_RET; WT_WITH_TABLE_WRITE_LOCK(session, - WT_WITH_TXN_ISOLATION( - session, WT_ISO_READ_UNCOMMITTED, ret = __schema_open_table(session, cfg))); + WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, ret = __schema_open_table(session))); return (ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index de902b8d442..f8c529ca74e 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -879,6 +879,77 @@ done: } /* + * __txn_commit_timestamps_usage_check -- + * Print warning messages when encountering unexpected timestamp usage. + */ +static inline int +__txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_UPDATE *upd) +{ + WT_TXN *txn; + wt_timestamp_t op_ts, prev_op_durable_ts; + uint32_t ts_flags; + char ts_string[2][WT_TS_INT_STRING_SIZE]; + bool txn_has_ts; + + txn = session->txn; + txn_has_ts = F_ISSET(txn, WT_TXN_HAS_TS_COMMIT | WT_TXN_HAS_TS_DURABLE); + +#define WT_COMMIT_TS_VERB_PREFIX "Commit timestamp unexpected usage: " + + /* If this transaction did not touch any table configured for verbose logging, we're done. */ + if (!F_ISSET(txn, WT_TXN_VERB_TS_WRITE)) + return (0); + + op_ts = upd->start_ts != WT_TS_NONE ? upd->start_ts : txn->commit_timestamp; + ts_flags = op->btree->dhandle->ts_flags; + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ALWAYS) && !txn_has_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "commit timestamp not used on table configured to require timestamps")); + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_NEVER) && txn_has_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "commit timestamp %s used on table configured to not use timestamps", + __wt_timestamp_to_string(op_ts, ts_string[0]))); + +#ifdef HAVE_DIAGNOSTIC + prev_op_durable_ts = upd->prev_durable_ts; + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_KEY_CONSISTENT) && prev_op_durable_ts != WT_TS_NONE && + !txn_has_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "no timestamp provided for an update to a " + "table configured to always use timestamps once they are first used")); + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && txn_has_ts && prev_op_durable_ts > op_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "committing a transaction that updates a " + "value with an older timestamp (%s) than is associated with the previous " + "update (%s) on a table configured for strict ordering", + __wt_timestamp_to_string(op_ts, ts_string[0]), + __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_MIXED_MODE) && F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && + op_ts != WT_TS_NONE && prev_op_durable_ts > op_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "committing a transaction that updates a " + "value with an older timestamp (%s) than is associated with the previous " + "update (%s) on a table configured for mixed mode ordering", + __wt_timestamp_to_string(op_ts, ts_string[0]), + __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); +#else + WT_UNUSED(prev_op_durable_ts); +#endif + + return (0); +} + +/* * __txn_fixup_prepared_update -- * Fix/restore the history store update of a prepared datastore update based on transaction * status. @@ -1043,6 +1114,7 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, if (upd == NULL || upd->prepare_state != WT_PREPARE_INPROGRESS) return (0); + WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); /* * Retrieve the previous update from the history store and append it to the update chain. * @@ -1167,35 +1239,31 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) WT_TXN *txn; WT_TXN_OP *op; WT_UPDATE *upd; - wt_timestamp_t durable_op_timestamp, op_timestamp, prev_op_timestamp; + wt_timestamp_t op_ts, prev_op_durable_ts, prev_op_ts; u_int i; - bool op_zero_ts, upd_zero_ts; + bool op_zero_ts, upd_zero_ts, used_ts; txn = session->txn; cursor = NULL; + used_ts = F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) || F_ISSET(txn, WT_TXN_HAS_TS_DURABLE); /* * Debugging checks on timestamps, if user requested them. */ - if (F_ISSET(txn, WT_TXN_TS_COMMIT_ALWAYS) && !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && - txn->mod_count != 0) + if (F_ISSET(txn, WT_TXN_TS_WRITE_ALWAYS) && !used_ts && txn->mod_count != 0) WT_RET_MSG(session, EINVAL, "commit_timestamp required and none set on this transaction"); - if (F_ISSET(txn, WT_TXN_TS_COMMIT_NEVER) && F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && - txn->mod_count != 0) + if (F_ISSET(txn, WT_TXN_TS_WRITE_NEVER) && used_ts && txn->mod_count != 0) + WT_RET_MSG( + session, EINVAL, "no commit_timestamp expected and timestamp set on this transaction"); + + if (txn->commit_timestamp > txn->durable_timestamp) WT_RET_MSG( - session, EINVAL, "no commit_timestamp required and timestamp set on this transaction"); - if (F_ISSET(txn, WT_TXN_TS_DURABLE_ALWAYS) && !F_ISSET(txn, WT_TXN_HAS_TS_DURABLE) && - txn->mod_count != 0) - WT_RET_MSG(session, EINVAL, "durable_timestamp required and none set on this transaction"); - if (F_ISSET(txn, WT_TXN_TS_DURABLE_NEVER) && F_ISSET(txn, WT_TXN_HAS_TS_DURABLE) && - txn->mod_count != 0) - WT_RET_MSG(session, EINVAL, - "no durable_timestamp required and durable timestamp set on this transaction"); + session, EINVAL, "transaction with commit timestamp greater than durable timestamp"); /* * If we're not doing any key consistency checking, we're done. */ - if (!F_ISSET(txn, WT_TXN_TS_COMMIT_KEYS | WT_TXN_TS_DURABLE_KEYS)) + if (!F_ISSET(txn, WT_TXN_TS_WRITE_KEY_CONSISTENT)) return (0); /* @@ -1222,7 +1290,7 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) else upd = op->u.op_upd; - op_timestamp = upd->start_ts; + op_ts = upd->start_ts; /* * Skip over any aborted update structures, internally created update structures or ones @@ -1232,6 +1300,11 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) (upd->txnid == WT_TXN_ABORTED || upd->txnid == WT_TXN_NONE || upd->txnid == txn->id)) upd = upd->next; + /* + * If we didn't track timestamps during update creation, and there are no more updates on + * the chain we won't check any further here. It's not worth reading updates from the disk + * to do this diagnostic checking. + */ if (upd == NULL) continue; @@ -1239,8 +1312,8 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) * Check the timestamp on this update with the first valid update in the chain. They're in * most recent order. */ - prev_op_timestamp = upd->start_ts; - durable_op_timestamp = upd->durable_ts; + prev_op_ts = upd->start_ts; + prev_op_durable_ts = upd->durable_ts; /* * Check for consistent per-key timestamp usage. If timestamps are or are not used @@ -1255,7 +1328,7 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) * by a failed eviction which attempted to write a prepared update to the data store. */ op_zero_ts = !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT); - upd_zero_ts = prev_op_timestamp == WT_TS_NONE; + upd_zero_ts = prev_op_durable_ts == WT_TS_NONE; if (op_zero_ts != upd_zero_ts && !F_ISSET(upd, WT_UPDATE_RESTORED_FROM_HS | WT_UPDATE_RESTORED_FROM_DS)) { WT_ERR(__wt_verbose_dump_update(session, upd)); @@ -1273,14 +1346,22 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) * Only if the update structure doesn't have a timestamp then use the one in the transaction * structure. */ - if (op_timestamp == WT_TS_NONE) - op_timestamp = txn->commit_timestamp; - if (F_ISSET(txn, WT_TXN_TS_COMMIT_KEYS) && op_timestamp < prev_op_timestamp) + if (op_ts == WT_TS_NONE) + op_ts = txn->commit_timestamp; + /* + * Check based on the durable timestamp, but first ensure that it's a stronger check than + * comparing commit timestamps would be. + */ + WT_ASSERT(session, txn->durable_timestamp >= op_ts && prev_op_durable_ts >= prev_op_ts); + if (F_ISSET(txn, WT_TXN_TS_WRITE_KEY_CONSISTENT) && + txn->durable_timestamp < prev_op_durable_ts) WT_ERR_MSG(session, EINVAL, "out of order commit timestamps"); - if (F_ISSET(txn, WT_TXN_TS_DURABLE_KEYS) && txn->durable_timestamp < durable_op_timestamp) - WT_ERR_MSG(session, EINVAL, "out of order durable timestamps"); } +#ifndef HAVE_DIAGNOSTIC + WT_UNUSED(prev_op_ts); +#endif + err: if (cursor != NULL) WT_TRET(cursor->close(cursor)); @@ -1490,6 +1571,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) break; __wt_txn_op_set_timestamp(session, op); + WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); } else { /* * If an operation has the key repeated flag set, skip resolving prepared updates as diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index bb6f1ff6c0f..f2320daf0ec 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -36,8 +36,7 @@ __rollback_abort_newer_update(WT_SESSION_IMPL *session, WT_UPDATE *first_upd, * here. */ WT_ASSERT(session, - !FLD_ISSET(S2BT(session)->assert_flags, WT_ASSERT_COMMIT_TS_KEYS) || - upd == first_upd); + !F_ISSET(session->dhandle, WT_DHANDLE_TS_KEY_CONSISTENT) || upd == first_upd); first_upd = upd->next; __wt_verbose(session, WT_VERB_RTS, diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index 2cece102729..dcd01e68dd4 100755 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -388,7 +388,7 @@ dump_projection(WT_SESSION *session, const char *config, WT_CURSOR *cursor, char while ((ret = parser->next(parser, &key, &value)) == 0) { WT_RET(dump_add_config(session, &newconfig, &len, "%.*s=", (int)key.len, key.str)); if (STRING_MATCH_CONFIG("value_format", key)) - WT_RET(dump_add_config(session, &newconfig, &len, "%s", cursor->value_format)); + WT_RET(dump_add_config(session, &newconfig, &len, "%s,", cursor->value_format)); else if (STRING_MATCH_CONFIG("columns", key)) { /* copy names of keys */ p = value.str; diff --git a/src/third_party/wiredtiger/test/cppsuite/Makefile.am b/src/third_party/wiredtiger/test/cppsuite/Makefile.am new file mode 100644 index 00000000000..4e819c3baea --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/Makefile.am @@ -0,0 +1,18 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility +LDADD = $(top_builddir)/test/utility/libtest_util.la \ + $(top_builddir)/libwiredtiger.la +AM_LDFLAGS = -static + +all_TESTS= +noinst_PROGRAMS= + +test_poc_SOURCES = tests/poc.cxx +noinst_PROGRAMS += test_poc +all_TESTS += test_poc + +# Run this during a "make check" smoke test. +TESTS = $(all_TESTS) + +clean-local: + rm -rf WT_TEST.* core.* *.core diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx new file mode 100644 index 00000000000..a087f44c514 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx @@ -0,0 +1,26 @@ +#include <iostream> +#include <stdlib.h> + +extern "C" { + #include "wiredtiger.h" +} + +int main(int argc, char *argv[]) { + WT_CONNECTION *conn; + int ret = 0; + /* Setup basic test directory. */ + const std::string default_dir = "WT_TEST"; + + /* + * Csuite tests utilise a test_util.h command to make their directory, currently that doesn't + * compile under c++ and some extra work will be needed to make it work. Its unclear if the + * test framework will use test_util.h yet. + */ + const std::string mkdir_cmd = "mkdir " + default_dir; + ret = system(mkdir_cmd.c_str()); + if (ret != 0) + return (ret); + + ret = wiredtiger_open(default_dir.c_str(), NULL, "create,cache_size=1G", &conn); + return (ret); +} diff --git a/src/third_party/wiredtiger/test/evergreen/evg_cfg.py b/src/third_party/wiredtiger/test/evergreen/evg_cfg.py index c9f8d591643..2caeda39101 100755 --- a/src/third_party/wiredtiger/test/evergreen/evg_cfg.py +++ b/src/third_party/wiredtiger/test/evergreen/evg_cfg.py @@ -30,6 +30,7 @@ CSUITE_TEST_SEARCH_STR = " # End of csuite test tasks" # They are not expected to trigger any 'make check' testing. make_check_subdir_skips = [ "test/csuite", # csuite has its own set of Evergreen tasks, skip the checking here + "test/cppsuite" ] prog=sys.argv[0] diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 5cb994e5c81..8ecccc2b87a 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -398,6 +398,12 @@ config_backward_compatible(void) EINVAL, "stress.hs_checkpoint_delay not supported in backward compatibility mode"); config_single("stress.hs_checkpoint_delay=off", false); } + + if (g.c_timing_stress_hs_search) { + if (config_is_perm("stress.hs_search")) + testutil_die(EINVAL, "stress.hs_search not supported in backward compatibility mode"); + config_single("stress.hs_search=off", false); + } } /* diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 321dc33ddd2..17f44648645 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -302,6 +302,10 @@ static CONFIG c[] = { &g.c_timing_stress_hs_checkpoint_delay, NULL}, /* 2% */ + {"stress.hs_search", "stress history store search", C_BOOL, 2, 0, 0, &g.c_timing_stress_hs_search, + NULL}, + + /* 2% */ {"stress.hs_sweep", "stress history store sweep", C_BOOL, 2, 0, 0, &g.c_timing_stress_hs_sweep, NULL}, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 27ed68ece19..26ada2deda7 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -215,6 +215,7 @@ typedef struct { uint32_t c_timing_stress_aggressive_sweep; uint32_t c_timing_stress_checkpoint; uint32_t c_timing_stress_hs_checkpoint_delay; + uint32_t c_timing_stress_hs_search; uint32_t c_timing_stress_hs_sweep; uint32_t c_timing_stress_checkpoint_prepare; uint32_t c_timing_stress_split_1; diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 7a91de6d10d..7b37f91573b 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -1355,6 +1355,9 @@ order_error_col: } if (0) { order_error_row: +#ifdef HAVE_DIAGNOSTIC + testutil_check(__wt_debug_cursor_page(cursor, g.home_pagedump)); +#endif testutil_die(0, "%s returned {%.*s} then {%.*s}", which, (int)tinfo->key->size, (char *)tinfo->key->data, (int)key.size, (char *)key.data); } diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index d017d28ff38..a586abe5ee6 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -232,6 +232,8 @@ create_database(const char *home, WT_CONNECTION **connp) CONFIG_APPEND(p, ",prepare_checkpoint_delay"); if (g.c_timing_stress_hs_checkpoint_delay) CONFIG_APPEND(p, ",history_store_checkpoint_delay"); + if (g.c_timing_stress_hs_search) + CONFIG_APPEND(p, ",history_store_search"); if (g.c_timing_stress_hs_sweep) CONFIG_APPEND(p, ",history_store_sweep_race"); if (g.c_timing_stress_split_1) @@ -362,7 +364,7 @@ create_object(WT_CONNECTION *conn) * Assertions. Assertions slow down the code for additional diagnostic checking. */ if (g.c_txn_timestamps && g.c_assert_commit_timestamp) - CONFIG_APPEND(p, ",assert=(commit_timestamp=key_consistent)"); + CONFIG_APPEND(p, ",write_timestamp_usage=key_consistent,assert=(write_timestamp=on)"); if (g.c_txn_timestamps && g.c_assert_read_timestamp) CONFIG_APPEND(p, ",assert=(read_timestamp=always)"); diff --git a/src/third_party/wiredtiger/test/suite/test_assert01.py b/src/third_party/wiredtiger/test/suite/test_assert01.py index 3b990d7eda7..228e56aef96 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert01.py +++ b/src/third_party/wiredtiger/test/suite/test_assert01.py @@ -44,10 +44,10 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' cfg = 'key_format=S,value_format=S,' - cfg_always = 'assert=(commit_timestamp=always)' + cfg_always = 'verbose=[write_timestamp],write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' - cfg_never = 'assert=(commit_timestamp=never)' - cfg_none = 'assert=(commit_timestamp=none)' + cfg_never = 'verbose=(write_timestamp=true),write_timestamp_usage=never,assert=(write_timestamp=on)' + cfg_none = 'assert=(write_timestamp=off)' session_config = 'isolation=snapshot' count = 1 @@ -94,9 +94,6 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_commit_timestamp(self): - #if not wiredtiger.diagnostic_build(): - # self.skipTest('requires a diagnostic build') - # Create a data item at a timestamp self.session.create(self.uri_always, self.cfg + self.cfg_always) self.session.create(self.uri_def, self.cfg + self.cfg_def) diff --git a/src/third_party/wiredtiger/test/suite/test_assert02.py b/src/third_party/wiredtiger/test/suite/test_assert02.py index eb67511772a..5091c83a8d5 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert02.py +++ b/src/third_party/wiredtiger/test/suite/test_assert02.py @@ -51,7 +51,7 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): uri_none = base_uri + '.none.wt' cfg = 'key_format=S,value_format=S' - cfg_always = cfg + ',assert=(read_timestamp=always)' + cfg_always = cfg + ',write_timestamp_usage=always,assert=(read_timestamp=always)' cfg_def = cfg cfg_never = cfg + ',assert=(read_timestamp=never)' cfg_none = cfg + ',assert=(read_timestamp=none)' diff --git a/src/third_party/wiredtiger/test/suite/test_assert03.py b/src/third_party/wiredtiger/test/suite/test_assert03.py index 7f929022c4b..63e29384e29 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert03.py +++ b/src/third_party/wiredtiger/test/suite/test_assert03.py @@ -37,9 +37,9 @@ class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'log=(enabled)' base_uri = 'file:assert03.wt' cfg = 'key_format=S,value_format=S' - always = 'assert=(commit_timestamp=always)' - never = 'assert=(commit_timestamp=never)' - none = 'assert=(commit_timestamp=none)' + always = 'write_timestamp_usage=always,assert=(write_timestamp=on)' + never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' + none = 'assert=(write_timestamp=off)' def test_assert03(self): #if not wiredtiger.diagnostic_build(): diff --git a/src/third_party/wiredtiger/test/suite/test_assert04.py b/src/third_party/wiredtiger/test/suite/test_assert04.py index 28cf0f2ab65..8a7637dd9e7 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert04.py +++ b/src/third_party/wiredtiger/test/suite/test_assert04.py @@ -42,8 +42,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): def test_timestamp_alter(self): base = 'assert04' uri = 'file:' + base - cfg_on = 'assert=(commit_timestamp=key_consistent)' - cfg_off = 'assert=(commit_timestamp=none)' + cfg_on = 'write_timestamp_usage=key_consistent,assert=(write_timestamp=on)' + cfg_off = 'assert=(write_timestamp=off)' msg_ooo='/out of order/' msg_usage='/used inconsistently/' @@ -179,7 +179,7 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,assert=(commit_timestamp=key_consistent)') + self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_assert05.py b/src/third_party/wiredtiger/test/suite/test_assert05.py index 3822fceb239..1a888bf1463 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert05.py +++ b/src/third_party/wiredtiger/test/suite/test_assert05.py @@ -45,10 +45,10 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' cfg = 'key_format=S,value_format=S,' - cfg_always = 'assert=(durable_timestamp=always)' + cfg_always = 'verbose=(write_timestamp=true),write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' - cfg_never = 'assert=(durable_timestamp=never)' - cfg_none = 'assert=(durable_timestamp=none)' + cfg_never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' + cfg_none = 'assert=(write_timestamp=off)' count = 1 # @@ -98,7 +98,7 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(self.count)) # All settings other than always should commit successfully - if (use_ts != 'always'): + if (use_ts != 'always' and use_ts != 'never'): self.session.commit_transaction() else: ''' @@ -114,9 +114,6 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_durable_timestamp(self): - #if not wiredtiger.diagnostic_build(): - # self.skipTest('requires a diagnostic build') - # Create a data item at a timestamp self.session.create(self.uri_always, self.cfg + self.cfg_always) self.session.create(self.uri_def, self.cfg + self.cfg_def) diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 4ae45b6dd61..ad2f6c345b1 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -50,8 +50,8 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): def test_timestamp_alter(self): base = 'assert06' uri = 'file:' + base - cfg_on = 'assert=(durable_timestamp=key_consistent)' - cfg_off = 'assert=(durable_timestamp=none)' + cfg_on = 'write_timestamp_usage=key_consistent,assert=(write_timestamp=on)' + cfg_off = 'write_timestamp_usage=never,assert=(write_timestamp=off)' msg_ooo='/out of order/' msg_usage='/used inconsistently/' @@ -189,7 +189,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,assert=(durable_timestamp=key_consistent)') + self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py new file mode 100755 index 00000000000..b9bc9777954 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_timestamp22.py +# Misuse the timestamp API, making sure we don't crash. +import wiredtiger, wttest, re, suite_random +from wtdataset import SimpleDataSet +from contextlib import contextmanager + +def timestamp_str(t): + return '%x' % t + +class test_timestamp22(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB' + session_config = 'isolation=snapshot' + + # Keep the number of rows low, as each additional row does + # not test any new code paths. + nrows = 3 + uri = "table:test_timestamp22" + rand = suite_random.suite_random() + oldest_ts = 0 + stable_ts = 0 + SUCCESS = 'success' + FAILURE = 'failure' + + # Control execution of an operation, looking for exceptions and error messages. + # Usage: + # with self.expect(self.FAILURE, 'some operation'): + # some_operation() # In this case, we expect it will fail + # + # "expected" argument can be self.SUCCESS, self.FAILURE, True, False, for convenience. + @contextmanager + def expect(self, expected, message): + if expected == True: + expected = self.SUCCESS + elif expected == False: + expected = self.FAILURE + + self.pr('TRYING: ' + message + ', expect ' + expected) + got = None + # If there are stray error messages from a previous operation, + # let's find out now. It can be confusing if we do something illegal + # here and we have multiple messages to sort out. + self.checkStderr() + + # 'yield' runs the subordinate operation, we'll catch any resulting exceptions. + try: + if expected == self.FAILURE: + # Soak up any error messages that happen as a result of the failure. + with self.expectedStderrPattern(r'^.*$', re_flags=re.MULTILINE): + yield + else: + yield + got = self.SUCCESS + except: + got = self.FAILURE + self.cleanStderr() + + message += ' got ' + got + + # If we're about to assert, show some extra info + if expected != got: + message += ': ERROR expected ' + expected + self.checkStderr() + self.pr(message) + self.assertEquals(expected, got) + + # Create a predictable value based on the iteration number and timestamp. + def gen_value(self, iternum, ts): + return str(iternum) + '_' + str(ts) + '_' + 'x' * 1000 + + # Given a number representing an "approximate timestamp", generate a timestamp + # that is near that number, either plus or minus. + def gen_ts(self, approx_ts): + # a number between -10 and 10: + n = self.rand.rand32() % 21 - 10 + ts = approx_ts + n + if ts <= 0: + ts = 1 + return ts + + # Asks whether we should do an illegal operation now. Return yes 5%. + def do_illegal(self): + return self.rand.rand32() % 20 == 0 + + def report(self, func, arg = None): + self.pr('DOING: ' + func + ('' if arg == None else '(' + arg + ')')) + + # Insert a set of rows, each insert in its own transaction, with the + # given timestamps. + def updates(self, value, ds, do_prepare, commit_ts, durable_ts, read_ts): + + # Generate a configuration for a timestamp_transaction() call. + # Returns: 1) whether it expects success, 2) config 3) new running commit timestamp + def timestamp_txn_config(commit_ts, running_commit_ts): + ok = True + config = '' + this_commit_ts = -1 + if self.do_illegal(): + # setting durable timestamp must be after prepare call + config += ',durable_timestamp=' + timestamp_str(self.gen_ts(commit_ts)) + ok = False + + # ODDITY: if we set the durable timestamp (which is illegal at this point), and set a + # valid commit timestamp, the timestamp_transaction() call will fail, but apparently, + # only the durable part fails. The evidence is that the commit timestamp is set, + # as we get a complaint to that effect at the prepare call. The issue is described + # in WT-6995. This seems wrong, and it's hard to work around the problem, so we'll just + # avoid testing that situation for now. Hence the check for a blank configuration. + # When WT-6995 is fixed, remove the "and config = ''" part of the clause immediately + # below, and this entire comment. + if self.rand.rand32() % 2 == 0 and config == '': + if self.do_illegal(): + this_commit_ts = self.oldest_ts - 1 + elif self.do_illegal(): + this_commit_ts = self.stable_ts - 1 + else: + # It's possible this will succeed, we'll check below. + this_commit_ts = self.gen_ts(commit_ts) + config += ',commit_timestamp=' + timestamp_str(this_commit_ts) + + if this_commit_ts >= 0: + if this_commit_ts < running_commit_ts: + ok = False + if this_commit_ts < self.stable_ts: + ok = False + if this_commit_ts < self.oldest_ts: + ok = False + if not ok: + this_commit_ts = -1 + if this_commit_ts >= 0: + running_commit_ts = this_commit_ts + return (ok, config, running_commit_ts) + + session = self.session + needs_rollback = False + prepare_config = None + commit_config = 'commit_timestamp=' + timestamp_str(commit_ts) + tstxn1_config = '' + tstxn2_config = '' + + ok_commit = do_prepare or not self.do_illegal() + ok_prepare = True + ok_tstxn1 = True + ok_tstxn2 = True + + # Occasionally put a durable timestamp on a commit without a prepare, + # that will be an error. + if do_prepare or not ok_commit: + commit_config += ',durable_timestamp=' + timestamp_str(durable_ts) + cursor = session.open_cursor(self.uri) + prepare_ts = self.gen_ts(commit_ts) + prepare_config = 'prepare_timestamp=' + timestamp_str(prepare_ts) + begin_config = '' if read_ts < 0 else 'read_timestamp=' + timestamp_str(read_ts) + + # We might do timestamp_transaction calls either before/after inserting + # values, or both. + do_tstxn1 = (self.rand.rand32() % 10 == 0) + do_tstxn2 = (self.rand.rand32() % 10 == 0) + + # Keep track of the commit timestamp that we'll set through the transaction. + # If it decreases, it will trigger an error. At the final commit_transaction + # operation, we'll use the commit_ts. + running_commit_ts = -1 + first_commit_ts = -1 + + if do_tstxn1: + (ok_tstxn1, tstxn1_config, running_commit_ts) = \ + timestamp_txn_config(commit_ts, running_commit_ts) + if first_commit_ts < 0: + first_commit_ts = running_commit_ts + + if do_tstxn2: + (ok_tstxn2, tstxn2_config, running_commit_ts) = \ + timestamp_txn_config(commit_ts, running_commit_ts) + if first_commit_ts < 0: + first_commit_ts = running_commit_ts + + # WT-7011: + # ODDITY: If any setting of the timestamp fails, then an ASSERT will be hit in prepare. + # Avoid this, it will crash the test suite when diagnostic mode is enabled, and without + # diagnostic mode, the prepare appears to succeed! Comment out the statement marked + # "AVOID ASSERT" below to see it happen. We should fix this to not assert in prepare, + # and either return an error in prepare, or fully succeed (forgiving the previous + # bad timestamp_transaction and allowing subsequent commit). If the former, + # then just remove the "AVOID ASSERT" line below. If the latter, then remove + # the entire if statement enclosing the "AVOID ASSERT". + if not ok_tstxn1 or not ok_tstxn2: + # If a setting of the timestamp fails, the prepare and commit both fail. + ok_prepare = False + ok_commit = False + do_prepare = False # AVOID ASSERT + + if running_commit_ts >= 0 and do_prepare: + # Cannot set prepare timestamp after commit timestamp is successfully set. + ok_prepare = False + + if do_prepare: + if commit_ts < prepare_ts: + ok_commit = False + if prepare_ts < self.oldest_ts: + ok_prepare = False + + # If the final commit is too old, we'll fail. + if commit_ts < self.oldest_ts or commit_ts < self.stable_ts: + ok_commit = False + + # ODDITY: We don't have to move the commit_ts ahead, but it has to be + # at least the value of the first commit timestamp set. + if commit_ts < first_commit_ts: + ok_commit = False + + # If a prepare fails, the commit fails as well. + if not ok_prepare: + ok_commit = False + + msg = 'inserts with commit config(' + commit_config + ')' + + try: + for i in range(0, self.nrows): + needs_rollback = False + if self.do_illegal(): + # Illegal outside of transaction + self.report('prepare_transaction', prepare_config) + with self.expect(False, 'prepare outside of transaction'): + session.prepare_transaction(prepare_config) + + with self.expect(True, 'begin_transaction(' + begin_config + ')'): + session.begin_transaction() + needs_rollback = True + + if do_tstxn1: + with self.expect(ok_tstxn1, 'timestamp_transaction(' + tstxn1_config + ')'): + session.timestamp_transaction(tstxn1_config) + + self.report('set key/value') + with self.expect(True, 'cursor insert'): + cursor[ds.key(i)] = value + + if do_tstxn2: + with self.expect(ok_tstxn2, 'timestamp_transaction(' + tstxn2_config + ')'): + session.timestamp_transaction(tstxn2_config) + + if do_prepare: + self.report('prepare_transaction', prepare_config) + with self.expect(ok_prepare, 'prepare'): + session.prepare_transaction(prepare_config) + + # Doing anything else after the prepare, like a timestamp_transaction(), will fail + # with a WT panic. Don't do that, or else we can't do anything more in this test. + + # If we did a successful prepare and are set up (by virtue of bad timestamps) + # to do a bad commit, WT will panic, and the test cannot continue. + # Only proceed with the commit if we have don't have that particular case. + if ok_commit or not do_prepare or not ok_prepare: + needs_rollback = False + self.report('commit_transaction', commit_config) + with self.expect(ok_commit, 'commit'): + session.commit_transaction(commit_config) + self.commit_value = value + if needs_rollback: + # Rollback this one transaction, and continue the loop + self.report('rollback_transaction') + needs_rollback = False + session.rollback_transaction() + except Exception as e: + # We don't expect any exceptions, they should be caught as part of self.expect statements. + self.pr(msg + 'UNEXPECTED EXCEPTION!') + self.pr(msg + 'fail: ' + str(e)) + raise e + cursor.close() + + def make_timestamp_config(self, oldest, stable, commit, durable): + configs = [] + # Get list of 'oldest_timestamp=value' etc. that have non-negative values. + for ts_name in ['oldest', 'stable', 'commit', 'durable']: + val = eval(ts_name) + if val >= 0: + configs.append(ts_name + '_timestamp=' + timestamp_str(val)) + return ','.join(configs) + + # Determine whether we expect the set_timestamp to succeed. + def expected_result_set_timestamp(self, oldest, stable, commit, durable): + + # Update the current expected value. ts is the timestamp being set. + # If "ts" is negative, ignore it, it's not being set in this call. + # It is unexpected if "ts" is before the "before" timestamp. + # The "before" timestamp could be updated during this call + # with value "before_arg", if not, use the global value for "before". + def expected_newer(expected, ts, before_arg, before_global): + if expected and ts >= 0: + if before_arg >= 0: + if before_arg > ts: + expected = self.FAILURE + else: + if before_global > ts: + expected = self.FAILURE + return expected + + expected = self.SUCCESS + if oldest >= 0 and stable < 0: + expected = expected_newer(expected, self.stable_ts, oldest, self.oldest_ts) + expected = expected_newer(expected, stable, oldest, self.oldest_ts) + expected = expected_newer(expected, commit, oldest, self.oldest_ts) + expected = expected_newer(expected, commit, stable, self.stable_ts) + + # If commit timestamp is set, durable timestamp is ignored. This seems to be + # a temporary situation, see TODO in txn_timestamp.c. + if commit < 0: + expected = expected_newer(expected, durable, oldest, self.oldest_ts) + expected = expected_newer(expected, durable, stable, self.stable_ts) + + return expected + + def set_global_timestamps(self, oldest, stable, commit, durable): + config = self.make_timestamp_config(oldest, stable, commit, durable) + expected = self.expected_result_set_timestamp(oldest, stable, commit, durable) + + with self.expect(expected, 'set_timestamp(' + config + ')'): + self.conn.set_timestamp(config) + + # Predict what we expect to happen to the timestamps. + if expected == self.SUCCESS: + # If that passes, then independently, oldest and stable can advance, but if they + # are less than the current value, that is silently ignored. + if oldest >= self.oldest_ts: + self.oldest_ts = oldest + self.pr('updating oldest: ' + str(oldest)) + if stable >= self.stable_ts: + self.stable_ts = stable + self.pr('updating stable: ' + str(stable)) + + # Make sure the state of global timestamps is what we think. + expect_query_oldest = timestamp_str(self.oldest_ts) + expect_query_stable = timestamp_str(self.stable_ts) + query_oldest = self.conn.query_timestamp('get=oldest') + query_stable = self.conn.query_timestamp('get=stable') + + self.assertEquals(expect_query_oldest, query_oldest) + self.assertEquals(expect_query_stable, query_stable) + self.pr('oldest now: ' + query_oldest) + self.pr('stable now: ' + query_stable) + + if expected == self.FAILURE: + self.cleanStderr() + + def test_timestamp_randomizer(self): + # Local function to generate a random timestamp, or return -1 + def maybe_ts(do_gen, iternum): + if do_gen: + return self.gen_ts(iternum) + else: + return -1 + + if wttest.islongtest(): + iterations = 100000 + else: + iterations = 1000 + + create_params = 'value_format=S,key_format=i' + self.session.create(self.uri, create_params) + + self.set_global_timestamps(1, 1, -1, -1) + + # Create tables with no entries + ds = SimpleDataSet( + self, self.uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + + # We do a bunch of iterations, doing transactions, prepare, and global timestamp calls + # with timestamps that are sometimes valid, sometimes not. We use the iteration number + # as an "approximate timestamp", and generate timestamps for our calls that are near + # that number (within 10). Thus, as the test runs, the timestamps generally get larger. + # We always know the state of global timestamps, so we can predict the success/failure + # on each call. + self.commit_value = '<NOT_SET>' + for iternum in range(1, iterations): + self.pr('\n===== ITERATION ' + str(iternum) + '/' + str(iterations)) + self.pr('RANDOM: ({0},{1})'.format(self.rand.seedw,self.rand.seedz)) + if self.rand.rand32() % 10 != 0: + commit_ts = self.gen_ts(iternum) + durable_ts = self.gen_ts(iternum) + do_prepare = (self.rand.rand32() % 20 == 0) + if self.rand.rand32() % 2 == 0: + read_ts = self.gen_ts(iternum) + else: + read_ts = -1 # no read_timestamp used in txn + if do_prepare: + # If we doing a prepare, we must abide by some additional rules. + # If we don't we'll immediately panic + if commit_ts < self.oldest_ts: + commit_ts = self.oldest_ts + if durable_ts < commit_ts: + durable_ts = commit_ts + value = self.gen_value(iternum, commit_ts) + self.updates(value, ds, do_prepare, commit_ts, durable_ts, read_ts) + + if self.rand.rand32() % 2 == 0: + # Set some combination of the global timestamps + r = self.rand.rand32() % 16 + oldest = maybe_ts((r & 0x1) != 0, iternum) + stable = maybe_ts((r & 0x2) != 0, iternum) + commit = maybe_ts((r & 0x4) != 0, iternum) + durable = maybe_ts((r & 0x8) != 0, iternum) + self.set_global_timestamps(oldest, stable, commit, durable) + + # Make sure the resulting rows are what we expect. + cursor = self.session.open_cursor(self.uri) + expect_key = 0 + expect_value = self.commit_value + for k,v in cursor: + self.assertEquals(k, expect_key) + self.assertEquals(v, expect_value) + expect_key += 1 + + # Although it's theoretically possible to never successfully update a single row, + # with a large number of iterations that should never happen. I'd rather catch + # a test code error where we mistakenly don't update any rows. + self.assertGreater(expect_key, 0) + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 0e7bb3ae6f9..0d11bfb193b 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -113,6 +113,14 @@ class CapturedFd(object): contents + '"') self.expectpos = os.path.getsize(self.filename) + def ignorePreviousOutput(self): + """ + Ignore any output up to this point. + """ + if self.file != None: + self.file.flush() + self.expectpos = os.path.getsize(self.filename) + def checkAdditional(self, testcase, expect): """ Check to see that an additional string has been added to the @@ -127,7 +135,7 @@ class CapturedFd(object): gotstr + '"') self.expectpos = os.path.getsize(self.filename) - def checkAdditionalPattern(self, testcase, pat): + def checkAdditionalPattern(self, testcase, pat, re_flags = 0): """ Check to see that an additional string has been added to the output file. If it has not, raise it as a test failure. @@ -136,7 +144,7 @@ class CapturedFd(object): if self.file != None: self.file.flush() gotstr = self.readFileFrom(self.filename, self.expectpos, 1500) - if re.search(pat, gotstr) == None: + if re.search(pat, gotstr, re_flags) == None: testcase.fail('in ' + self.desc + ', expected pattern "' + pat + '", but got "' + gotstr + '"') @@ -419,6 +427,18 @@ class WiredTigerTestCase(unittest.TestCase): if exc_list and exc_list[-1][0] is self: return exc_list[-1][1] + def cleanStderr(self): + self.captureerr.ignorePreviousOutput() + + def cleanStdout(self): + self.captureout.ignorePreviousOutput() + + def checkStderr(self): + self.captureerr.check(self) + + def checkStdout(self): + self.captureout.check(self) + def tearDown(self): # This approach works for all our support Python versions and # is suggested by one of the answers in: @@ -512,24 +532,24 @@ class WiredTigerTestCase(unittest.TestCase): self.captureerr.checkAdditional(self, expect) @contextmanager - def expectedStdoutPattern(self, pat): + def expectedStdoutPattern(self, pat, re_flags=0): self.captureout.check(self) yield - self.captureout.checkAdditionalPattern(self, pat) + self.captureout.checkAdditionalPattern(self, pat, re_flags) @contextmanager - def expectedStderrPattern(self, pat): + def expectedStderrPattern(self, pat, re_flags=0): self.captureerr.check(self) yield - self.captureerr.checkAdditionalPattern(self, pat) + self.captureerr.checkAdditionalPattern(self, pat, re_flags) - def ignoreStdoutPatternIfExists(self, pat): + def ignoreStdoutPatternIfExists(self, pat, re_flags=0): if self.captureout.hasUnexpectedOutput(self): - self.captureout.checkAdditionalPattern(self, pat) + self.captureout.checkAdditionalPattern(self, pat, re_flags) - def ignoreStderrPatternIfExists(self, pat): + def ignoreStderrPatternIfExists(self, pat, re_flags=0): if self.captureerr.hasUnexpectedOutput(self): - self.captureerr.checkAdditionalPattern(self, pat) + self.captureerr.checkAdditionalPattern(self, pat, re_flags) def assertRaisesWithMessage(self, exceptionType, expr, message): """ |