diff options
| author | Chenhao Qu <chenhao.qu@mongodb.com> | 2020-12-16 05:41:30 +0000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-12-16 06:19:38 +0000 |
| commit | 6019169e828c7100cd72ac617ab0ce579b20c957 (patch) | |
| tree | fcaf0c62fb06d3246bba807f804dfb0d3c6e6eca /src | |
| parent | 356787c1f465d35fa17aa2e7e6fd7e2ba09ce998 (diff) | |
| download | mongo-6019169e828c7100cd72ac617ab0ce579b20c957.tar.gz | |
Import wiredtiger: 34b59326932d1d0f00dd8559cdb70db23ebbc330 from branch mongodb-5.0
ref: d4cbdb9907..34b5932693
for: 4.9.0
WT-6432 Add test case to misuse the timestamp API
WT-6846 Initial test program for the new cpp test framework
WT-6861 Add the ability to log messages about unexpected timestamp usage
WT-6994 Dump the cursor page whenever a key out order is detected
Diffstat (limited to 'src')
37 files changed, 1028 insertions, 268 deletions
diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index da149c335f4..117e54bb613 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -29,6 +29,7 @@ examples/c # Test programs. test/bloom test/checkpoint +test/cppsuite HAVE_CXX test/csuite test/cursor_order test/fops diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 5986a676926..957ef40ff09 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -45,6 +45,43 @@ class Config: common_runtime_config = [ Config('app_metadata', '', r''' application-owned metadata for this object'''), + Config('assert', '', r''' + enable enhanced checking. ''', + type='category', subconfig= [ + Config('write_timestamp', 'off', r''' + verify that commit timestamps are used per the configured + \c write_timestamp_usage option for this table''', + choices=['off', 'on']), + Config('read_timestamp', 'none', r''' + verify that timestamps should \c always or \c never be used + on reads with this table. Verification is \c none + if mixed read use is allowed''', + choices=['always', 'never', 'none']) + ], undoc=True), + Config('verbose', '[]', r''' + enable messages for various events. Options are given as a + list, such as <code>"verbose=[write_timestamp]"</code>''', + type='list', choices=[ + 'write_timestamp', + ]), + Config('write_timestamp_usage', 'none', r''' + describe how timestamps are expected to be used on modifications to + the table. This option should be used in conjunction with the + corresponding \c write_timestamp configuration under the \c assert and + \c verbose options to provide logging and assertions for incorrect + timestamp usage. The choices are \c always which ensures a timestamp is + used for every operation on a table, \c key_consistent to ensure that + once timestamps are used for a key, they are always used, \c ordered is + like \c key_consistent except it also enforces that subsequent updates + to each key must use increasing timestamps, \c mixed_mode is like + \c ordered except that updates with no timestamp are allowed and have + the effect of resetting the chain of updates once the transaction ID + based snapshot is no longer relevant, \c never enforces that timestamps + are never used for a table and \c none does not enforce any expectation + on timestamp usage meaning that no log message or assertions will be + produced regardless of the corresponding \c assert and \c verbose + settings''', + choices=['always', 'key_consistent', 'mixed_mode', 'never', 'none', 'ordered']), ] # Metadata shared by all schema objects @@ -169,26 +206,6 @@ file_runtime_config = common_runtime_config + [ option leads to an advisory call to an appropriate operating system API where available''', choices=['none', 'random', 'sequential']), - Config('assert', '', r''' - enable enhanced checking. ''', - type='category', subconfig= [ - Config('commit_timestamp', 'none', r''' - verify that timestamps should 'always' or 'never' be used - on modifications with this table. Verification is 'none' - if mixed update use is allowed. If 'key_consistent' is - set then all updates to a specific key must be the same - with respect to timestamp usage or not.''', - choices=['always', 'key_consistent', 'never', 'none']), - Config('durable_timestamp', 'none', r''' - verify that durable timestamps should 'always' or 'never' be used - on modifications with this table.''', - choices=['always', 'key_consistent', 'never', 'none']), - Config('read_timestamp', 'none', r''' - verify that timestamps should 'always' or 'never' be used - on reads with this table. Verification is 'none' - if mixed read use is allowed.''', - choices=['always', 'never', 'none']) - ], undoc=True), Config('cache_resident', 'false', r''' do not ever evict the object's pages from cache. Not compatible with LSM tables; see @ref tuning_cache_resident for more information''', @@ -697,7 +714,7 @@ connection_runtime_config = [ 'aggressive_sweep', 'backup_rename', 'checkpoint_slow', 'history_store_checkpoint_delay', 'history_store_search', 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', 'split_2', 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']), - Config('verbose', '', r''' + Config('verbose', '[]', r''' enable messages for various events. Options are given as a list, such as <code>"verbose=[evictserver,read]"</code>''', type='list', choices=[ diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 1445a734185..323791f09f8 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "d4cbdb99074600e2b12153fbfab790349e6bdd0a" + "commit": "34b59326932d1d0f00dd8559cdb70db23ebbc330" } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 00ae7165454..1801394cf27 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -444,6 +444,8 @@ __cursor_key_order_check_col(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo return (0); } + WT_RET(__wt_msg(session, "dumping the cursor page")); + WT_RET(__wt_debug_cursor_page(&cbt->iface, NULL)); WT_RET_PANIC(session, EINVAL, "WT_CURSOR.%s out-of-order returns: returned key %" PRIu64 " then key %" PRIu64, next ? "next" : "prev", cbt->lastrecno, cbt->recno); @@ -476,6 +478,8 @@ __cursor_key_order_check_row(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, boo WT_ERR(__wt_scr_alloc(session, 512, &a)); WT_ERR(__wt_scr_alloc(session, 512, &b)); + WT_ERR(__wt_msg(session, "dumping the cursor page")); + WT_ERR(__wt_debug_cursor_page(&cbt->iface, NULL)); WT_ERR_PANIC(session, EINVAL, "WT_CURSOR.%s out-of-order returns: returned key %.1024s then key %.1024s", next ? "next" : "prev", diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index e5e2efada27..23a9c4f8078 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -921,7 +921,7 @@ __curfile_update_check(WT_CURSOR_BTREE *cbt) else if (btree->type != BTREE_COL_VAR) return (0); - return (__wt_txn_update_check(session, cbt, upd)); + return (__wt_txn_update_check(session, cbt, upd, NULL)); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 60cde6ce8b5..c745a8078ea 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -408,34 +408,6 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) else btree->checksum = CKSUM_UNCOMPRESSED; - /* Debugging information */ - WT_RET(__wt_config_gets(session, cfg, "assert.commit_timestamp", &cval)); - btree->assert_flags = 0; - if (WT_STRING_MATCH("always", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_COMMIT_TS_ALWAYS); - else if (WT_STRING_MATCH("key_consistent", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_COMMIT_TS_KEYS); - else if (WT_STRING_MATCH("never", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_COMMIT_TS_NEVER); - - /* - * A durable timestamp always implies a commit timestamp. But never having a durable timestamp - * does not imply anything about a commit timestamp. - */ - WT_RET(__wt_config_gets(session, cfg, "assert.durable_timestamp", &cval)); - if (WT_STRING_MATCH("always", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_COMMIT_TS_ALWAYS | WT_ASSERT_DURABLE_TS_ALWAYS); - else if (WT_STRING_MATCH("key_consistent", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_DURABLE_TS_KEYS); - else if (WT_STRING_MATCH("never", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_DURABLE_TS_NEVER); - - WT_RET(__wt_config_gets(session, cfg, "assert.read_timestamp", &cval)); - if (WT_STRING_MATCH("always", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_READ_TS_ALWAYS); - else if (WT_STRING_MATCH("never", cval.str, cval.len)) - FLD_SET(btree->assert_flags, WT_ASSERT_READ_TS_NEVER); - /* Huffman encoding */ WT_RET(__wt_btree_huffman_open(session)); diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index 49e57bd51d8..6be6ea9589c 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -27,6 +27,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_PAGE_MODIFY *mod; WT_SESSION_IMPL *session; WT_UPDATE *old_upd, *upd; + wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; u_int i, skipdepth; bool append, logged; @@ -36,6 +37,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U page = cbt->ref->page; session = CUR2S(cbt); upd = upd_arg; + prev_upd_ts = WT_TS_NONE; append = logged = false; /* @@ -125,10 +127,13 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U old_upd = cbt->ins->upd; if (upd_arg == NULL) { /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd)); + WT_ERR(__wt_txn_update_check(session, cbt, old_upd, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); +#ifdef HAVE_DIAGNOSTIC + upd->prev_durable_ts = prev_upd_ts; +#endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; } else { diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index 652c59c71a0..ef600f7d88c 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -51,6 +51,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, WT_PAGE_MODIFY *mod; WT_SESSION_IMPL *session; WT_UPDATE *last_upd, *old_upd, *upd, **upd_entry; + wt_timestamp_t prev_upd_ts; size_t ins_size, upd_size; uint32_t ins_slot; u_int i, skipdepth; @@ -61,6 +62,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, session = CUR2S(cbt); last_upd = NULL; upd = upd_arg; + prev_upd_ts = WT_TS_NONE; inserted_to_update_chain = logged = false; /* @@ -103,10 +105,13 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, if (upd_arg == NULL) { /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check(session, cbt, old_upd = *upd_entry)); + WT_ERR(__wt_txn_update_check(session, cbt, old_upd = *upd_entry, &prev_upd_ts)); /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); +#ifdef HAVE_DIAGNOSTIC + upd->prev_durable_ts = prev_upd_ts; +#endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index ca6b04d2c2b..ef434fa3e5d 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -154,15 +154,8 @@ static const WT_CONFIG_CHECK confchk_WT_CURSOR_reconfigure[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_assert_subconfigs[] = { - {"commit_timestamp", "string", NULL, - "choices=[\"always\",\"key_consistent\",\"never\"," - "\"none\"]", - NULL, 0}, - {"durable_timestamp", "string", NULL, - "choices=[\"always\",\"key_consistent\",\"never\"," - "\"none\"]", - NULL, 0}, {"read_timestamp", "string", NULL, "choices=[\"always\",\"never\",\"none\"]", NULL, 0}, + {"write_timestamp", "string", NULL, "choices=[\"off\",\"on\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_create_log_subconfigs[] = { @@ -171,12 +164,18 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create_log_subconfigs[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"exclusive_refreshed", "boolean", NULL, NULL, NULL, 0}, {"log", "category", NULL, NULL, confchk_WT_SESSION_create_log_subconfigs, 1}, {"os_cache_dirty_max", "int", NULL, "min=0", NULL, 0}, - {"os_cache_max", "int", NULL, "min=0", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"os_cache_max", "int", NULL, "min=0", NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_begin_transaction_roundup_timestamps_subconfigs[] = {{"prepared", "boolean", NULL, NULL, NULL, 0}, {"read", "boolean", NULL, NULL, NULL, 0}, @@ -241,7 +240,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, @@ -276,6 +275,11 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_WT_SESSION_drop[] = { @@ -367,15 +371,22 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_verify[] = { {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_colgroup_meta[] = { - {"app_metadata", "string", NULL, NULL, NULL, 0}, {"collator", "string", NULL, NULL, NULL, 0}, - {"columns", "list", NULL, NULL, NULL, 0}, {"source", "string", NULL, NULL, NULL, 0}, - {"type", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"app_metadata", "string", NULL, NULL, NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, + {"source", "string", NULL, NULL, NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_file_config[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, @@ -406,13 +417,18 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_file_meta[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, @@ -445,22 +461,35 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, - {"version", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"version", "string", NULL, NULL, NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, + {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_index_meta[] = { - {"app_metadata", "string", NULL, NULL, NULL, 0}, {"collator", "string", NULL, NULL, NULL, 0}, - {"columns", "list", NULL, NULL, NULL, 0}, {"extractor", "string", NULL, NULL, NULL, 0}, - {"immutable", "boolean", NULL, NULL, NULL, 0}, {"index_key_columns", "int", NULL, NULL, NULL, 0}, + {"app_metadata", "string", NULL, NULL, NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, + {"extractor", "string", NULL, NULL, NULL, 0}, {"immutable", "boolean", NULL, NULL, NULL, 0}, + {"index_key_columns", "int", NULL, NULL, NULL, 0}, {"key_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"source", "string", NULL, NULL, NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"access_pattern_hint", "string", NULL, "choices=[\"none\",\"random\",\"sequential\"]", NULL, 0}, {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, - {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 3}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, {"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, @@ -493,13 +522,25 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"split_deepen_per_child", "int", NULL, NULL, NULL, 0}, {"split_pct", "int", NULL, "min=50,max=100", NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_table_meta[] = { - {"app_metadata", "string", NULL, NULL, NULL, 0}, {"colgroups", "list", NULL, NULL, NULL, 0}, - {"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0}, + {"app_metadata", "string", NULL, NULL, NULL, 0}, + {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 2}, + {"colgroups", "list", NULL, NULL, NULL, 0}, {"collator", "string", NULL, NULL, NULL, 0}, + {"columns", "list", NULL, NULL, NULL, 0}, {"key_format", "format", __wt_struct_confchk, NULL, NULL, 0}, {"value_format", "format", __wt_struct_confchk, NULL, NULL, 0}, + {"verbose", "list", NULL, "choices=[\"write_timestamp\"]", NULL, 0}, + {"write_timestamp_usage", "string", NULL, + "choices=[\"always\",\"key_consistent\",\"mixed_mode\"," + "\"never\",\"none\",\"ordered\"]", + NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_wiredtiger_open_compatibility_subconfigs[] = { @@ -865,7 +906,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(json=false,on_close=false," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," - "timing_stress_for_test=,verbose=", + "timing_stress_for_test=,verbose=[]", confchk_WT_CONNECTION_reconfigure, 27}, {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", @@ -876,11 +917,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_CURSOR.reconfigure", "append=false,overwrite=true", confchk_WT_CURSOR_reconfigure, 2}, {"WT_SESSION.alter", "access_pattern_hint=none,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),cache_resident=false," - "exclusive_refreshed=true,log=(enabled=true),os_cache_dirty_max=0" - ",os_cache_max=0", - confchk_WT_SESSION_alter, 8}, + "assert=(read_timestamp=none,write_timestamp=off)," + "cache_resident=false,exclusive_refreshed=true,log=(enabled=true)" + ",os_cache_dirty_max=0,os_cache_max=0,verbose=[]," + "write_timestamp_usage=none", + confchk_WT_SESSION_alter, 10}, {"WT_SESSION.begin_transaction", "ignore_prepare=false,isolation=,name=,operation_timeout_ms=0," "priority=0,read_before_oldest=false,read_timestamp=," @@ -896,14 +937,13 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.compact", "timeout=1200", confchk_WT_SESSION_compact, 1}, {"WT_SESSION.create", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checksum=uncompressed,colgroups=,collator=," - "columns=,dictionary=0,encryption=(keyid=,name=),exclusive=false," - "extractor=,format=btree,huffman_value=," - "ignore_in_memory_cache_size=false,immutable=false," - "import=(enabled=false,file_metadata=,repair=false)," - "internal_item_max=0,internal_key_max=0," + "assert=(read_timestamp=none,write_timestamp=off)," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,colgroups=,collator=,columns=,dictionary=0" + ",encryption=(keyid=,name=),exclusive=false,extractor=," + "format=btree,huffman_value=,ignore_in_memory_cache_size=false," + "immutable=false,import=(enabled=false,file_metadata=," + "repair=false),internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," @@ -914,8 +954,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," "prefix_compression=false,prefix_compression_min=4,source=," "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "type=file,value_format=u", - confchk_WT_SESSION_create, 44}, + "type=file,value_format=u,verbose=[],write_timestamp_usage=none", + confchk_WT_SESSION_create, 46}, {"WT_SESSION.drop", "checkpoint_wait=true,force=false,lock_wait=true," "remove_files=true", @@ -958,69 +998,75 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "dump_offsets=,dump_pages=false,stable_timestamp=false," "strict=false", confchk_WT_SESSION_verify, 7}, - {"colgroup.meta", "app_metadata=,collator=,columns=,source=,type=file", confchk_colgroup_meta, 5}, + {"colgroup.meta", + "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "collator=,columns=,source=,type=file,verbose=[]," + "write_timestamp_usage=none", + confchk_colgroup_meta, 8}, {"file.config", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checksum=uncompressed,collator=,columns=," - "dictionary=0,encryption=(keyid=,name=),format=btree," - "huffman_value=,ignore_in_memory_cache_size=false," - "internal_item_max=0,internal_key_max=0," - "internal_key_truncate=true,internal_page_max=4KB,key_format=u," - "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," - "leaf_value_max=0,log=(enabled=true),memory_page_image_max=0," - "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "value_format=u", - confchk_file_config, 36}, + "assert=(read_timestamp=none,write_timestamp=off)," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_value=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," + "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," + "log=(enabled=true),memory_page_image_max=0,memory_page_max=5MB," + "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," + "prefix_compression_min=4,split_deepen_min_child=0," + "split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]," + "write_timestamp_usage=none", + confchk_file_config, 38}, {"file.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checkpoint=,checkpoint_backup_info=," - "checkpoint_lsn=,checksum=uncompressed,collator=,columns=," - "dictionary=0,encryption=(keyid=,name=),format=btree," - "huffman_value=,id=,ignore_in_memory_cache_size=false," - "internal_item_max=0,internal_key_max=0," - "internal_key_truncate=true,internal_page_max=4KB,key_format=u," - "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," - "leaf_value_max=0,log=(enabled=true),memory_page_image_max=0," - "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "value_format=u,version=(major=0,minor=0)", - confchk_file_meta, 41}, + "assert=(read_timestamp=none,write_timestamp=off)," + "block_allocation=best,block_compressor=,cache_resident=false," + "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=," + "checksum=uncompressed,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_value=,id=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," + "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," + "log=(enabled=true),memory_page_image_max=0,memory_page_max=5MB," + "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," + "prefix_compression_min=4,split_deepen_min_child=0," + "split_deepen_per_child=0,split_pct=90,value_format=u,verbose=[]," + "version=(major=0,minor=0),write_timestamp_usage=none", + confchk_file_meta, 43}, {"index.meta", - "app_metadata=,collator=,columns=,extractor=,immutable=false," - "index_key_columns=,key_format=u,source=,type=file,value_format=u", - confchk_index_meta, 10}, + "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "collator=,columns=,extractor=,immutable=false,index_key_columns=" + ",key_format=u,source=,type=file,value_format=u,verbose=[]," + "write_timestamp_usage=none", + confchk_index_meta, 13}, {"lsm.meta", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," - "assert=(commit_timestamp=none,durable_timestamp=none," - "read_timestamp=none),block_allocation=best,block_compressor=," - "cache_resident=false,checksum=uncompressed,chunks=,collator=," - "columns=,dictionary=0,encryption=(keyid=,name=),format=btree," - "huffman_value=,ignore_in_memory_cache_size=false," - "internal_item_max=0,internal_key_max=0," - "internal_key_truncate=true,internal_page_max=4KB,key_format=u," - "key_gap=10,last=,leaf_item_max=0,leaf_key_max=0," - "leaf_page_max=32KB,leaf_value_max=0,log=(enabled=true)," - "lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16," - "bloom_config=,bloom_hash_count=8,bloom_oldest=false," - "chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB," - "merge_custom=(prefix=,start_generation=0,suffix=),merge_max=15," - "merge_min=0),memory_page_image_max=0,memory_page_max=5MB," - "old_chunks=,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," - "value_format=u", - confchk_lsm_meta, 40}, + "assert=(read_timestamp=none,write_timestamp=off)," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,chunks=,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_value=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,last=," + "leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," + "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," + "bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8," + "bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB," + "chunk_size=10MB,merge_custom=(prefix=,start_generation=0," + "suffix=),merge_max=15,merge_min=0),memory_page_image_max=0," + "memory_page_max=5MB,old_chunks=,os_cache_dirty_max=0," + "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" + ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "value_format=u,verbose=[],write_timestamp_usage=none", + confchk_lsm_meta, 42}, {"table.meta", - "app_metadata=,colgroups=,collator=,columns=,key_format=u," - "value_format=u", - confchk_table_meta, 6}, + "app_metadata=,assert=(read_timestamp=none,write_timestamp=off)," + "colgroups=,collator=,columns=,key_format=u,value_format=u," + "verbose=[],write_timestamp_usage=none", + confchk_table_meta, 9}, {"wiredtiger_open", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" ",cache_max_wait_ms=0,cache_overhead=8,cache_size=100MB," @@ -1051,7 +1097,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," - "verbose=,verify_metadata=false,write_through=", + "verbose=[],verify_metadata=false,write_through=", confchk_wiredtiger_open, 55}, {"wiredtiger_open_all", "buffer_alignment=-1,builtin_extension_config=,cache_cursors=true" @@ -1083,7 +1129,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),timing_stress_for_test=,transaction_sync=(enabled=false" ",method=fsync),use_environment=true,use_environment_priv=false," - "verbose=,verify_metadata=false,version=(major=0,minor=0)," + "verbose=[],verify_metadata=false,version=(major=0,minor=0)," "write_through=", confchk_wiredtiger_open_all, 56}, {"wiredtiger_open_basecfg", @@ -1114,7 +1160,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=,verify_metadata=false,version=(major=0," + "method=fsync),verbose=[],verify_metadata=false,version=(major=0," "minor=0),write_through=", confchk_wiredtiger_open_basecfg, 50}, {"wiredtiger_open_usercfg", @@ -1145,7 +1191,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=,verify_metadata=false,write_through=", + "method=fsync),verbose=[],verify_metadata=false,write_through=", confchk_wiredtiger_open_usercfg, 49}, {NULL, NULL, NULL, 0}}; diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index fdcc6f4b944..1d792d74163 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1834,12 +1834,6 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) return (0); } -/* Simple structure for name and flag configuration searches. */ -typedef struct { - const char *name; - uint64_t flag; -} WT_NAME_FLAG; - /* * __wt_verbose_config -- * Set verbose configuration. diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 16a9d7812b5..44b2d86f7c5 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -410,6 +410,57 @@ err: } /* + * __conn_dhandle_config_parse -- + * Parse out configuration settings relevant for the data handle. + */ +static int +__conn_dhandle_config_parse(WT_SESSION_IMPL *session) +{ + WT_CONFIG_ITEM cval, sval; + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; + const char **cfg; + + dhandle = session->dhandle; + cfg = dhandle->cfg; + + /* Clear all the flags. */ + dhandle->ts_flags = 0; + + /* Debugging information */ + WT_RET(__wt_config_gets(session, cfg, "assert.write_timestamp", &cval)); + if (WT_STRING_MATCH("on", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_WRITE); + + WT_RET(__wt_config_gets(session, cfg, "assert.read_timestamp", &cval)); + if (WT_STRING_MATCH("always", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_ALWAYS); + else if (WT_STRING_MATCH("never", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_NEVER); + + /* Setup verbose options */ + WT_RET(__wt_config_gets(session, cfg, "verbose", &cval)); + if ((ret = __wt_config_subgets(session, &cval, "write_timestamp", &sval)) == 0 && sval.val != 0) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_VERB_TS_WRITE); + WT_RET_NOTFOUND_OK(ret); + + /* Setup timestamp usage hints */ + WT_RET(__wt_config_gets(session, cfg, "write_timestamp_usage", &cval)); + if (WT_STRING_MATCH("always", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_ALWAYS); + else if (WT_STRING_MATCH("key_consistent", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_KEY_CONSISTENT); + else if (WT_STRING_MATCH("mixed_mode", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_MIXED_MODE); + else if (WT_STRING_MATCH("never", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_NEVER); + else if (WT_STRING_MATCH("ordered", cval.str, cval.len)) + FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_ORDERED); + + return (0); +} + +/* * __wt_conn_dhandle_open -- * Open the current data handle. */ @@ -447,6 +498,7 @@ __wt_conn_dhandle_open(WT_SESSION_IMPL *session, const char *cfg[], uint32_t fla /* Discard any previous configuration, set up the new configuration. */ __conn_dhandle_config_clear(session); WT_ERR(__conn_dhandle_config_set(session)); + WT_ERR(__conn_dhandle_config_parse(session)); switch (dhandle->type) { case WT_DHANDLE_TYPE_BTREE: @@ -465,7 +517,7 @@ __wt_conn_dhandle_open(WT_SESSION_IMPL *session, const char *cfg[], uint32_t fla WT_ERR(__wt_btree_open(session, cfg)); break; case WT_DHANDLE_TYPE_TABLE: - WT_ERR(__wt_schema_open_table(session, cfg)); + WT_ERR(__wt_schema_open_table(session)); break; } diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 3cc12d78e74..2d7008d3383 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -1054,6 +1054,9 @@ struct __wt_update { wt_timestamp_t durable_ts; /* timestamps */ wt_timestamp_t start_ts; +#ifdef HAVE_DIAGNOSTIC + wt_timestamp_t prev_durable_ts; +#endif WT_UPDATE *next; /* forward-linked list */ @@ -1099,7 +1102,11 @@ struct __wt_update { * WT_UPDATE_SIZE is the expected structure size excluding the payload data -- we verify the build * to ensure the compiler hasn't inserted padding. */ +#ifdef HAVE_DIAGNOSTIC +#define WT_UPDATE_SIZE 47 +#else #define WT_UPDATE_SIZE 39 +#endif /* * The memory size of an update: include some padding because this is such a common case that diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 4af6ea90f67..85ddcff102d 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -100,18 +100,6 @@ struct __wt_btree { uint32_t maxmempage_image; /* In-memory page image max size */ uint64_t splitmempage; /* In-memory split trigger size */ -/* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_ASSERT_COMMIT_TS_ALWAYS 0x01u -#define WT_ASSERT_COMMIT_TS_KEYS 0x02u -#define WT_ASSERT_COMMIT_TS_NEVER 0x04u -#define WT_ASSERT_DURABLE_TS_ALWAYS 0x08u -#define WT_ASSERT_DURABLE_TS_KEYS 0x10u -#define WT_ASSERT_DURABLE_TS_NEVER 0x20u -#define WT_ASSERT_READ_TS_ALWAYS 0x40u -#define WT_ASSERT_READ_TS_NEVER 0x80u - /* AUTOMATIC FLAG VALUE GENERATION STOP */ - uint32_t assert_flags; /* Debugging assertion information */ - void *huffman_value; /* Value huffman encoding */ enum { diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 79b786b25dc..d52fe680d81 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -102,6 +102,15 @@ struct __wt_named_extractor { }; /* + * WT_NAME_FLAG -- + * Simple structure for name and flag configuration searches + */ +struct __wt_name_flag { + const char *name; + uint64_t flag; +}; + +/* * WT_CONN_CHECK_PANIC -- * Check if we've panicked and return the appropriate error. */ diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 49e9e756fe6..2ee23b71701 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -112,4 +112,17 @@ struct __wt_data_handle { #define WT_DHANDLE_OPEN 0x80u /* Handle is open */ /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_DHANDLE_ASSERT_TS_READ_ALWAYS 0x001u /* Assert read always checking. */ +#define WT_DHANDLE_ASSERT_TS_READ_NEVER 0x002u /* Assert read never checking. */ +#define WT_DHANDLE_ASSERT_TS_WRITE 0x004u /* Assert write checking. */ +#define WT_DHANDLE_TS_ALWAYS 0x008u /* Handle using always checking. */ +#define WT_DHANDLE_TS_KEY_CONSISTENT 0x010u /* Handle using key consistency checking. */ +#define WT_DHANDLE_TS_MIXED_MODE 0x020u /* Handle using mixed mode timestamps checking. */ +#define WT_DHANDLE_TS_NEVER 0x040u /* Handle never using timestamps checking. */ +#define WT_DHANDLE_TS_ORDERED 0x080u /* Handle using ordered timestamps checking. */ +#define WT_DHANDLE_VERB_TS_WRITE 0x100u /* Handle verbose logging for timestamps usage. */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint32_t ts_flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index d902d72ff01..1c646d6622c 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1289,7 +1289,7 @@ extern int __wt_schema_open_index(WT_SESSION_IMPL *session, WT_TABLE *table, con size_t len, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_indices(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) +extern int __wt_schema_open_table(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_project_in(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -2048,7 +2048,7 @@ static inline int __wt_txn_read_upd_list(WT_SESSION_IMPL *session, WT_CURSOR_BTR static inline int __wt_txn_search_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + WT_UPDATE *upd, wt_timestamp_t *prev_tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, u_int modify_type, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp, diff --git a/src/third_party/wiredtiger/src/include/serial_inline.h b/src/third_party/wiredtiger/src/include/serial_inline.h index 11e49c9081a..a469b0db0f6 100644 --- a/src/third_party/wiredtiger/src/include/serial_inline.h +++ b/src/third_party/wiredtiger/src/include/serial_inline.h @@ -222,12 +222,13 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page { WT_DECL_RET; WT_UPDATE *obsolete, *upd; - wt_timestamp_t obsolete_timestamp; + wt_timestamp_t obsolete_timestamp, prev_upd_ts; uint64_t txn; /* Clear references to memory we now own and must free on error. */ upd = *updp; *updp = NULL; + prev_upd_ts = WT_TS_NONE; /* * All structure setup must be flushed before the structure is entered into the list. We need a @@ -237,12 +238,15 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page * Check if our update is still permitted. */ while (!__wt_atomic_cas_ptr(srch_upd, upd->next, upd)) { - if ((ret = __wt_txn_update_check(session, cbt, upd->next = *srch_upd)) != 0) { + if ((ret = __wt_txn_update_check(session, cbt, upd->next = *srch_upd, &prev_upd_ts)) != 0) { /* Free unused memory on error. */ __wt_free(session, upd); return (ret); } } +#ifdef HAVE_DIAGNOSTIC + upd->prev_durable_ts = prev_upd_ts; +#endif /* * Increment in-memory footprint after swapping the update into place. Safe because the diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index c3b7f37d8fe..ddfcf99a52d 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -340,16 +340,16 @@ struct __wt_txn { #define WT_TXN_SHARED_TS_DURABLE 0x000800u #define WT_TXN_SHARED_TS_READ 0x001000u #define WT_TXN_SYNC_SET 0x002000u -#define WT_TXN_TS_COMMIT_ALWAYS 0x004000u -#define WT_TXN_TS_COMMIT_KEYS 0x008000u -#define WT_TXN_TS_COMMIT_NEVER 0x010000u -#define WT_TXN_TS_DURABLE_ALWAYS 0x020000u -#define WT_TXN_TS_DURABLE_KEYS 0x040000u -#define WT_TXN_TS_DURABLE_NEVER 0x080000u -#define WT_TXN_TS_READ_BEFORE_OLDEST 0x100000u -#define WT_TXN_TS_ROUND_PREPARED 0x200000u -#define WT_TXN_TS_ROUND_READ 0x400000u -#define WT_TXN_UPDATE 0x800000u +#define WT_TXN_TS_READ_BEFORE_OLDEST 0x004000u +#define WT_TXN_TS_ROUND_PREPARED 0x008000u +#define WT_TXN_TS_ROUND_READ 0x010000u +#define WT_TXN_TS_WRITE_ALWAYS 0x020000u +#define WT_TXN_TS_WRITE_KEY_CONSISTENT 0x040000u +#define WT_TXN_TS_WRITE_MIXED_MODE 0x080000u +#define WT_TXN_TS_WRITE_NEVER 0x100000u +#define WT_TXN_TS_WRITE_ORDERED 0x200000u +#define WT_TXN_UPDATE 0x400000u +#define WT_TXN_VERB_TS_WRITE 0x800000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index 3afd00eb76c..0433a252d63 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -71,24 +71,32 @@ static inline void __wt_txn_timestamp_flags(WT_SESSION_IMPL *session) { WT_BTREE *btree; + WT_DATA_HANDLE *dhandle; - if (session->dhandle == NULL) + dhandle = session->dhandle; + if (dhandle == NULL) return; btree = S2BT(session); if (btree == NULL) return; - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_COMMIT_TS_ALWAYS)) - F_SET(session->txn, WT_TXN_TS_COMMIT_ALWAYS); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_COMMIT_TS_KEYS)) - F_SET(session->txn, WT_TXN_TS_COMMIT_KEYS); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_COMMIT_TS_NEVER)) - F_SET(session->txn, WT_TXN_TS_COMMIT_NEVER); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_DURABLE_TS_ALWAYS)) - F_SET(session->txn, WT_TXN_TS_DURABLE_ALWAYS); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_DURABLE_TS_KEYS)) - F_SET(session->txn, WT_TXN_TS_DURABLE_KEYS); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_DURABLE_TS_NEVER)) - F_SET(session->txn, WT_TXN_TS_DURABLE_NEVER); + + if (!FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_WRITE)) + return; + + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_ALWAYS)) + F_SET(session->txn, WT_TXN_TS_WRITE_ALWAYS); + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_KEY_CONSISTENT)) + F_SET(session->txn, WT_TXN_TS_WRITE_KEY_CONSISTENT); + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_MIXED_MODE)) + F_SET(session->txn, WT_TXN_TS_WRITE_MIXED_MODE); + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_NEVER)) + F_SET(session->txn, WT_TXN_TS_WRITE_NEVER); + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_ORDERED)) + F_SET(session->txn, WT_TXN_TS_WRITE_ORDERED); + + /* Remember if any type of verbose tracking is encountered by the transaction. */ + if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_VERB_TS_WRITE)) + F_SET(session->txn, WT_TXN_VERB_TS_WRITE); } /* @@ -1214,10 +1222,10 @@ __wt_txn_search_check(WT_SESSION_IMPL *session) * Same if it should never have a read timestamp. */ if (!F_ISSET(S2C(session), WT_CONN_RECOVERING) && - FLD_ISSET(btree->assert_flags, WT_ASSERT_READ_TS_ALWAYS) && + FLD_ISSET(btree->dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_ALWAYS) && !F_ISSET(txn, WT_TXN_SHARED_TS_READ)) WT_RET_MSG(session, EINVAL, "read_timestamp required and none set on this transaction"); - if (FLD_ISSET(btree->assert_flags, WT_ASSERT_READ_TS_NEVER) && + if (FLD_ISSET(btree->dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_NEVER) && F_ISSET(txn, WT_TXN_SHARED_TS_READ)) WT_RET_MSG( session, EINVAL, "no read_timestamp required and timestamp set on this transaction"); @@ -1229,7 +1237,8 @@ __wt_txn_search_check(WT_SESSION_IMPL *session) * Check if the current transaction can update an item. */ static inline int -__wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) +__wt_txn_update_check( + WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, wt_timestamp_t *prev_tsp) { WT_DECL_RET; WT_TIME_WINDOW tw; @@ -1282,6 +1291,16 @@ __wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE ret = __wt_txn_rollback_required(session, "conflict between concurrent operations"); } + if (prev_tsp != NULL && upd != NULL) { + /* + * The durable timestamp must be greater than or equal to the commit timestamp unless it is + * an in-progress prepared update. + * + * FIXME-WT-7020: We should be able to assert this but we're seeing some fallout in format. + * We should investigate why and assert the above statement. + */ + *prev_tsp = upd->durable_ts; + } if (ignore_prepare_set) F_SET(txn, WT_TXN_IGNORE_PREPARE); return (ret); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index f44b4e188c2..249e6629680 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1002,6 +1002,24 @@ struct __wt_session { * object blocks from the system buffer cache after that many bytes from this object are * read or written into the buffer cache., an integer greater than or equal to 0; default \c * 0.} + * @config{verbose, enable messages for various events. Options are given as a list\, such + * as <code>"verbose=[write_timestamp]"</code>., a list\, with values chosen from the + * following options: \c "write_timestamp"; default \c [].} + * @config{write_timestamp_usage, describe how timestamps are expected to be used on + * modifications to the table. This option should be used in conjunction with the + * corresponding \c write_timestamp configuration under the \c assert and \c verbose options + * to provide logging and assertions for incorrect timestamp usage. The choices are \c + * always which ensures a timestamp is used for every operation on a table\, \c + * key_consistent to ensure that once timestamps are used for a key\, they are always used\, + * \c ordered is like \c key_consistent except it also enforces that subsequent updates to + * each key must use increasing timestamps\, \c mixed_mode is like \c ordered except that + * updates with no timestamp are allowed and have the effect of resetting the chain of + * updates once the transaction ID based snapshot is no longer relevant\, \c never enforces + * that timestamps are never used for a table and \c none does not enforce any expectation + * on timestamp usage meaning that no log message or assertions will be produced regardless + * of the corresponding \c assert and \c verbose settings., a string\, chosen from the + * following options: \c "always"\, \c "key_consistent"\, \c "mixed_mode"\, \c "never"\, \c + * "none"\, \c "ordered"; default \c none.} * @configend * @errors */ @@ -1225,6 +1243,24 @@ struct __wt_session { * applications use a WT_ITEM structure to manipulate raw byte arrays. Value items of type * 't' are bitfields\, and when configured with record number type keys\, will be stored * using a fixed-length store., a format string; default \c u.} + * @config{verbose, enable messages for various events. Options are given as a list\, such + * as <code>"verbose=[write_timestamp]"</code>., a list\, with values chosen from the + * following options: \c "write_timestamp"; default \c [].} + * @config{write_timestamp_usage, describe how timestamps are expected to be used on + * modifications to the table. This option should be used in conjunction with the + * corresponding \c write_timestamp configuration under the \c assert and \c verbose options + * to provide logging and assertions for incorrect timestamp usage. The choices are \c + * always which ensures a timestamp is used for every operation on a table\, \c + * key_consistent to ensure that once timestamps are used for a key\, they are always used\, + * \c ordered is like \c key_consistent except it also enforces that subsequent updates to + * each key must use increasing timestamps\, \c mixed_mode is like \c ordered except that + * updates with no timestamp are allowed and have the effect of resetting the chain of + * updates once the transaction ID based snapshot is no longer relevant\, \c never enforces + * that timestamps are never used for a table and \c none does not enforce any expectation + * on timestamp usage meaning that no log message or assertions will be produced regardless + * of the corresponding \c assert and \c verbose settings., a string\, chosen from the + * following options: \c "always"\, \c "key_consistent"\, \c "mixed_mode"\, \c "never"\, \c + * "none"\, \c "ordered"; default \c none.} * @configend * @errors */ @@ -2139,7 +2175,7 @@ struct __wt_connection { * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c "reconcile"\, * \c "recovery"\, \c "recovery_progress"\, \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c * "split"\, \c "temporary"\, \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c - * "verify"\, \c "version"\, \c "write"; default empty.} + * "verify"\, \c "version"\, \c "write"; default \c [].} * @configend * @errors */ @@ -2856,7 +2892,7 @@ struct __wt_connection { * "history_store"\, \c "history_store_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c * "mutex"\, \c "overflow"\, \c "read"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, * \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, - * \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; default empty.} + * \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; default \c [].} * @config{verify_metadata, open connection and verify any WiredTiger metadata. This API allows * verification and detection of corruption in WiredTiger metadata., a boolean flag; default \c * false.} diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index a64fb5acdfe..d3aa5905d4c 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -241,6 +241,8 @@ struct __wt_multi; typedef struct __wt_multi WT_MULTI; struct __wt_myslot; typedef struct __wt_myslot WT_MYSLOT; +struct __wt_name_flag; +typedef struct __wt_name_flag WT_NAME_FLAG; struct __wt_named_collator; typedef struct __wt_named_collator WT_NAMED_COLLATOR; struct __wt_named_compressor; diff --git a/src/third_party/wiredtiger/src/schema/schema_open.c b/src/third_party/wiredtiger/src/schema/schema_open.c index 79a07f95912..3af4f12606a 100644 --- a/src/third_party/wiredtiger/src/schema/schema_open.c +++ b/src/third_party/wiredtiger/src/schema/schema_open.c @@ -393,7 +393,7 @@ __wt_schema_open_indices(WT_SESSION_IMPL *session, WT_TABLE *table) * Open the data handle for a table (internal version). */ static int -__schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) +__schema_open_table(WT_SESSION_IMPL *session) { WT_CONFIG cparser; WT_CONFIG_ITEM ckey, cval; @@ -407,7 +407,6 @@ __schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) tablename = table->iface.name; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); - WT_UNUSED(cfg); WT_RET(__wt_config_gets(session, table_cfg, "columns", &cval)); WT_RET(__wt_config_gets(session, table_cfg, "key_format", &cval)); @@ -551,13 +550,12 @@ err: * Open a named table. */ int -__wt_schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_schema_open_table(WT_SESSION_IMPL *session) { WT_DECL_RET; WT_WITH_TABLE_WRITE_LOCK(session, - WT_WITH_TXN_ISOLATION( - session, WT_ISO_READ_UNCOMMITTED, ret = __schema_open_table(session, cfg))); + WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, ret = __schema_open_table(session))); return (ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index de902b8d442..f8c529ca74e 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -879,6 +879,77 @@ done: } /* + * __txn_commit_timestamps_usage_check -- + * Print warning messages when encountering unexpected timestamp usage. + */ +static inline int +__txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_UPDATE *upd) +{ + WT_TXN *txn; + wt_timestamp_t op_ts, prev_op_durable_ts; + uint32_t ts_flags; + char ts_string[2][WT_TS_INT_STRING_SIZE]; + bool txn_has_ts; + + txn = session->txn; + txn_has_ts = F_ISSET(txn, WT_TXN_HAS_TS_COMMIT | WT_TXN_HAS_TS_DURABLE); + +#define WT_COMMIT_TS_VERB_PREFIX "Commit timestamp unexpected usage: " + + /* If this transaction did not touch any table configured for verbose logging, we're done. */ + if (!F_ISSET(txn, WT_TXN_VERB_TS_WRITE)) + return (0); + + op_ts = upd->start_ts != WT_TS_NONE ? upd->start_ts : txn->commit_timestamp; + ts_flags = op->btree->dhandle->ts_flags; + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ALWAYS) && !txn_has_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "commit timestamp not used on table configured to require timestamps")); + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_NEVER) && txn_has_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "commit timestamp %s used on table configured to not use timestamps", + __wt_timestamp_to_string(op_ts, ts_string[0]))); + +#ifdef HAVE_DIAGNOSTIC + prev_op_durable_ts = upd->prev_durable_ts; + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_KEY_CONSISTENT) && prev_op_durable_ts != WT_TS_NONE && + !txn_has_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "no timestamp provided for an update to a " + "table configured to always use timestamps once they are first used")); + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && txn_has_ts && prev_op_durable_ts > op_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "committing a transaction that updates a " + "value with an older timestamp (%s) than is associated with the previous " + "update (%s) on a table configured for strict ordering", + __wt_timestamp_to_string(op_ts, ts_string[0]), + __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); + + if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_MIXED_MODE) && F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && + op_ts != WT_TS_NONE && prev_op_durable_ts > op_ts) + WT_RET(__wt_msg(session, + WT_COMMIT_TS_VERB_PREFIX + "committing a transaction that updates a " + "value with an older timestamp (%s) than is associated with the previous " + "update (%s) on a table configured for mixed mode ordering", + __wt_timestamp_to_string(op_ts, ts_string[0]), + __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]))); +#else + WT_UNUSED(prev_op_durable_ts); +#endif + + return (0); +} + +/* * __txn_fixup_prepared_update -- * Fix/restore the history store update of a prepared datastore update based on transaction * status. @@ -1043,6 +1114,7 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, if (upd == NULL || upd->prepare_state != WT_PREPARE_INPROGRESS) return (0); + WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); /* * Retrieve the previous update from the history store and append it to the update chain. * @@ -1167,35 +1239,31 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) WT_TXN *txn; WT_TXN_OP *op; WT_UPDATE *upd; - wt_timestamp_t durable_op_timestamp, op_timestamp, prev_op_timestamp; + wt_timestamp_t op_ts, prev_op_durable_ts, prev_op_ts; u_int i; - bool op_zero_ts, upd_zero_ts; + bool op_zero_ts, upd_zero_ts, used_ts; txn = session->txn; cursor = NULL; + used_ts = F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) || F_ISSET(txn, WT_TXN_HAS_TS_DURABLE); /* * Debugging checks on timestamps, if user requested them. */ - if (F_ISSET(txn, WT_TXN_TS_COMMIT_ALWAYS) && !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && - txn->mod_count != 0) + if (F_ISSET(txn, WT_TXN_TS_WRITE_ALWAYS) && !used_ts && txn->mod_count != 0) WT_RET_MSG(session, EINVAL, "commit_timestamp required and none set on this transaction"); - if (F_ISSET(txn, WT_TXN_TS_COMMIT_NEVER) && F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && - txn->mod_count != 0) + if (F_ISSET(txn, WT_TXN_TS_WRITE_NEVER) && used_ts && txn->mod_count != 0) + WT_RET_MSG( + session, EINVAL, "no commit_timestamp expected and timestamp set on this transaction"); + + if (txn->commit_timestamp > txn->durable_timestamp) WT_RET_MSG( - session, EINVAL, "no commit_timestamp required and timestamp set on this transaction"); - if (F_ISSET(txn, WT_TXN_TS_DURABLE_ALWAYS) && !F_ISSET(txn, WT_TXN_HAS_TS_DURABLE) && - txn->mod_count != 0) - WT_RET_MSG(session, EINVAL, "durable_timestamp required and none set on this transaction"); - if (F_ISSET(txn, WT_TXN_TS_DURABLE_NEVER) && F_ISSET(txn, WT_TXN_HAS_TS_DURABLE) && - txn->mod_count != 0) - WT_RET_MSG(session, EINVAL, - "no durable_timestamp required and durable timestamp set on this transaction"); + session, EINVAL, "transaction with commit timestamp greater than durable timestamp"); /* * If we're not doing any key consistency checking, we're done. */ - if (!F_ISSET(txn, WT_TXN_TS_COMMIT_KEYS | WT_TXN_TS_DURABLE_KEYS)) + if (!F_ISSET(txn, WT_TXN_TS_WRITE_KEY_CONSISTENT)) return (0); /* @@ -1222,7 +1290,7 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) else upd = op->u.op_upd; - op_timestamp = upd->start_ts; + op_ts = upd->start_ts; /* * Skip over any aborted update structures, internally created update structures or ones @@ -1232,6 +1300,11 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) (upd->txnid == WT_TXN_ABORTED || upd->txnid == WT_TXN_NONE || upd->txnid == txn->id)) upd = upd->next; + /* + * If we didn't track timestamps during update creation, and there are no more updates on + * the chain we won't check any further here. It's not worth reading updates from the disk + * to do this diagnostic checking. + */ if (upd == NULL) continue; @@ -1239,8 +1312,8 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) * Check the timestamp on this update with the first valid update in the chain. They're in * most recent order. */ - prev_op_timestamp = upd->start_ts; - durable_op_timestamp = upd->durable_ts; + prev_op_ts = upd->start_ts; + prev_op_durable_ts = upd->durable_ts; /* * Check for consistent per-key timestamp usage. If timestamps are or are not used @@ -1255,7 +1328,7 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) * by a failed eviction which attempted to write a prepared update to the data store. */ op_zero_ts = !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT); - upd_zero_ts = prev_op_timestamp == WT_TS_NONE; + upd_zero_ts = prev_op_durable_ts == WT_TS_NONE; if (op_zero_ts != upd_zero_ts && !F_ISSET(upd, WT_UPDATE_RESTORED_FROM_HS | WT_UPDATE_RESTORED_FROM_DS)) { WT_ERR(__wt_verbose_dump_update(session, upd)); @@ -1273,14 +1346,22 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) * Only if the update structure doesn't have a timestamp then use the one in the transaction * structure. */ - if (op_timestamp == WT_TS_NONE) - op_timestamp = txn->commit_timestamp; - if (F_ISSET(txn, WT_TXN_TS_COMMIT_KEYS) && op_timestamp < prev_op_timestamp) + if (op_ts == WT_TS_NONE) + op_ts = txn->commit_timestamp; + /* + * Check based on the durable timestamp, but first ensure that it's a stronger check than + * comparing commit timestamps would be. + */ + WT_ASSERT(session, txn->durable_timestamp >= op_ts && prev_op_durable_ts >= prev_op_ts); + if (F_ISSET(txn, WT_TXN_TS_WRITE_KEY_CONSISTENT) && + txn->durable_timestamp < prev_op_durable_ts) WT_ERR_MSG(session, EINVAL, "out of order commit timestamps"); - if (F_ISSET(txn, WT_TXN_TS_DURABLE_KEYS) && txn->durable_timestamp < durable_op_timestamp) - WT_ERR_MSG(session, EINVAL, "out of order durable timestamps"); } +#ifndef HAVE_DIAGNOSTIC + WT_UNUSED(prev_op_ts); +#endif + err: if (cursor != NULL) WT_TRET(cursor->close(cursor)); @@ -1490,6 +1571,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) break; __wt_txn_op_set_timestamp(session, op); + WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); } else { /* * If an operation has the key repeated flag set, skip resolving prepared updates as diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index bb6f1ff6c0f..f2320daf0ec 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -36,8 +36,7 @@ __rollback_abort_newer_update(WT_SESSION_IMPL *session, WT_UPDATE *first_upd, * here. */ WT_ASSERT(session, - !FLD_ISSET(S2BT(session)->assert_flags, WT_ASSERT_COMMIT_TS_KEYS) || - upd == first_upd); + !F_ISSET(session->dhandle, WT_DHANDLE_TS_KEY_CONSISTENT) || upd == first_upd); first_upd = upd->next; __wt_verbose(session, WT_VERB_RTS, diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index 2cece102729..dcd01e68dd4 100755 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -388,7 +388,7 @@ dump_projection(WT_SESSION *session, const char *config, WT_CURSOR *cursor, char while ((ret = parser->next(parser, &key, &value)) == 0) { WT_RET(dump_add_config(session, &newconfig, &len, "%.*s=", (int)key.len, key.str)); if (STRING_MATCH_CONFIG("value_format", key)) - WT_RET(dump_add_config(session, &newconfig, &len, "%s", cursor->value_format)); + WT_RET(dump_add_config(session, &newconfig, &len, "%s,", cursor->value_format)); else if (STRING_MATCH_CONFIG("columns", key)) { /* copy names of keys */ p = value.str; diff --git a/src/third_party/wiredtiger/test/cppsuite/Makefile.am b/src/third_party/wiredtiger/test/cppsuite/Makefile.am new file mode 100644 index 00000000000..4e819c3baea --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/Makefile.am @@ -0,0 +1,18 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility +LDADD = $(top_builddir)/test/utility/libtest_util.la \ + $(top_builddir)/libwiredtiger.la +AM_LDFLAGS = -static + +all_TESTS= +noinst_PROGRAMS= + +test_poc_SOURCES = tests/poc.cxx +noinst_PROGRAMS += test_poc +all_TESTS += test_poc + +# Run this during a "make check" smoke test. +TESTS = $(all_TESTS) + +clean-local: + rm -rf WT_TEST.* core.* *.core diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx new file mode 100644 index 00000000000..a087f44c514 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/poc.cxx @@ -0,0 +1,26 @@ +#include <iostream> +#include <stdlib.h> + +extern "C" { + #include "wiredtiger.h" +} + +int main(int argc, char *argv[]) { + WT_CONNECTION *conn; + int ret = 0; + /* Setup basic test directory. */ + const std::string default_dir = "WT_TEST"; + + /* + * Csuite tests utilise a test_util.h command to make their directory, currently that doesn't + * compile under c++ and some extra work will be needed to make it work. Its unclear if the + * test framework will use test_util.h yet. + */ + const std::string mkdir_cmd = "mkdir " + default_dir; + ret = system(mkdir_cmd.c_str()); + if (ret != 0) + return (ret); + + ret = wiredtiger_open(default_dir.c_str(), NULL, "create,cache_size=1G", &conn); + return (ret); +} diff --git a/src/third_party/wiredtiger/test/evergreen/evg_cfg.py b/src/third_party/wiredtiger/test/evergreen/evg_cfg.py index c9f8d591643..2caeda39101 100755 --- a/src/third_party/wiredtiger/test/evergreen/evg_cfg.py +++ b/src/third_party/wiredtiger/test/evergreen/evg_cfg.py @@ -30,6 +30,7 @@ CSUITE_TEST_SEARCH_STR = " # End of csuite test tasks" # They are not expected to trigger any 'make check' testing. make_check_subdir_skips = [ "test/csuite", # csuite has its own set of Evergreen tasks, skip the checking here + "test/cppsuite" ] prog=sys.argv[0] diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index ada83e6ecb7..a586abe5ee6 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -364,7 +364,7 @@ create_object(WT_CONNECTION *conn) * Assertions. Assertions slow down the code for additional diagnostic checking. */ if (g.c_txn_timestamps && g.c_assert_commit_timestamp) - CONFIG_APPEND(p, ",assert=(commit_timestamp=key_consistent)"); + CONFIG_APPEND(p, ",write_timestamp_usage=key_consistent,assert=(write_timestamp=on)"); if (g.c_txn_timestamps && g.c_assert_read_timestamp) CONFIG_APPEND(p, ",assert=(read_timestamp=always)"); diff --git a/src/third_party/wiredtiger/test/suite/test_assert01.py b/src/third_party/wiredtiger/test/suite/test_assert01.py index 3b990d7eda7..228e56aef96 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert01.py +++ b/src/third_party/wiredtiger/test/suite/test_assert01.py @@ -44,10 +44,10 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' cfg = 'key_format=S,value_format=S,' - cfg_always = 'assert=(commit_timestamp=always)' + cfg_always = 'verbose=[write_timestamp],write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' - cfg_never = 'assert=(commit_timestamp=never)' - cfg_none = 'assert=(commit_timestamp=none)' + cfg_never = 'verbose=(write_timestamp=true),write_timestamp_usage=never,assert=(write_timestamp=on)' + cfg_none = 'assert=(write_timestamp=off)' session_config = 'isolation=snapshot' count = 1 @@ -94,9 +94,6 @@ class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_commit_timestamp(self): - #if not wiredtiger.diagnostic_build(): - # self.skipTest('requires a diagnostic build') - # Create a data item at a timestamp self.session.create(self.uri_always, self.cfg + self.cfg_always) self.session.create(self.uri_def, self.cfg + self.cfg_def) diff --git a/src/third_party/wiredtiger/test/suite/test_assert02.py b/src/third_party/wiredtiger/test/suite/test_assert02.py index eb67511772a..5091c83a8d5 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert02.py +++ b/src/third_party/wiredtiger/test/suite/test_assert02.py @@ -51,7 +51,7 @@ class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): uri_none = base_uri + '.none.wt' cfg = 'key_format=S,value_format=S' - cfg_always = cfg + ',assert=(read_timestamp=always)' + cfg_always = cfg + ',write_timestamp_usage=always,assert=(read_timestamp=always)' cfg_def = cfg cfg_never = cfg + ',assert=(read_timestamp=never)' cfg_none = cfg + ',assert=(read_timestamp=none)' diff --git a/src/third_party/wiredtiger/test/suite/test_assert03.py b/src/third_party/wiredtiger/test/suite/test_assert03.py index 7f929022c4b..63e29384e29 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert03.py +++ b/src/third_party/wiredtiger/test/suite/test_assert03.py @@ -37,9 +37,9 @@ class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): conn_config = 'log=(enabled)' base_uri = 'file:assert03.wt' cfg = 'key_format=S,value_format=S' - always = 'assert=(commit_timestamp=always)' - never = 'assert=(commit_timestamp=never)' - none = 'assert=(commit_timestamp=none)' + always = 'write_timestamp_usage=always,assert=(write_timestamp=on)' + never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' + none = 'assert=(write_timestamp=off)' def test_assert03(self): #if not wiredtiger.diagnostic_build(): diff --git a/src/third_party/wiredtiger/test/suite/test_assert04.py b/src/third_party/wiredtiger/test/suite/test_assert04.py index 28cf0f2ab65..8a7637dd9e7 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert04.py +++ b/src/third_party/wiredtiger/test/suite/test_assert04.py @@ -42,8 +42,8 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): def test_timestamp_alter(self): base = 'assert04' uri = 'file:' + base - cfg_on = 'assert=(commit_timestamp=key_consistent)' - cfg_off = 'assert=(commit_timestamp=none)' + cfg_on = 'write_timestamp_usage=key_consistent,assert=(write_timestamp=on)' + cfg_off = 'assert=(write_timestamp=off)' msg_ooo='/out of order/' msg_usage='/used inconsistently/' @@ -179,7 +179,7 @@ class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,assert=(commit_timestamp=key_consistent)') + self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_assert05.py b/src/third_party/wiredtiger/test/suite/test_assert05.py index 3822fceb239..1a888bf1463 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert05.py +++ b/src/third_party/wiredtiger/test/suite/test_assert05.py @@ -45,10 +45,10 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' cfg = 'key_format=S,value_format=S,' - cfg_always = 'assert=(durable_timestamp=always)' + cfg_always = 'verbose=(write_timestamp=true),write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' - cfg_never = 'assert=(durable_timestamp=never)' - cfg_none = 'assert=(durable_timestamp=none)' + cfg_never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' + cfg_none = 'assert=(write_timestamp=off)' count = 1 # @@ -98,7 +98,7 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(self.count)) # All settings other than always should commit successfully - if (use_ts != 'always'): + if (use_ts != 'always' and use_ts != 'never'): self.session.commit_transaction() else: ''' @@ -114,9 +114,6 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): c.close() def test_durable_timestamp(self): - #if not wiredtiger.diagnostic_build(): - # self.skipTest('requires a diagnostic build') - # Create a data item at a timestamp self.session.create(self.uri_always, self.cfg + self.cfg_always) self.session.create(self.uri_def, self.cfg + self.cfg_def) diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 4ae45b6dd61..ad2f6c345b1 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -50,8 +50,8 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): def test_timestamp_alter(self): base = 'assert06' uri = 'file:' + base - cfg_on = 'assert=(durable_timestamp=key_consistent)' - cfg_off = 'assert=(durable_timestamp=none)' + cfg_on = 'write_timestamp_usage=key_consistent,assert=(write_timestamp=on)' + cfg_off = 'write_timestamp_usage=never,assert=(write_timestamp=off)' msg_ooo='/out of order/' msg_usage='/used inconsistently/' @@ -189,7 +189,7 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): # That checking will verify any individual key is always or never # used with a timestamp. And if it is used with a timestamp that # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format=S,value_format=S,assert=(durable_timestamp=key_consistent)') + self.session.create(uri, 'key_format=S,value_format=S,write_timestamp_usage=key_consistent,assert=(write_timestamp=on)') # Insert a data item at timestamp 2. c = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py new file mode 100755 index 00000000000..b9bc9777954 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_timestamp22.py +# Misuse the timestamp API, making sure we don't crash. +import wiredtiger, wttest, re, suite_random +from wtdataset import SimpleDataSet +from contextlib import contextmanager + +def timestamp_str(t): + return '%x' % t + +class test_timestamp22(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB' + session_config = 'isolation=snapshot' + + # Keep the number of rows low, as each additional row does + # not test any new code paths. + nrows = 3 + uri = "table:test_timestamp22" + rand = suite_random.suite_random() + oldest_ts = 0 + stable_ts = 0 + SUCCESS = 'success' + FAILURE = 'failure' + + # Control execution of an operation, looking for exceptions and error messages. + # Usage: + # with self.expect(self.FAILURE, 'some operation'): + # some_operation() # In this case, we expect it will fail + # + # "expected" argument can be self.SUCCESS, self.FAILURE, True, False, for convenience. + @contextmanager + def expect(self, expected, message): + if expected == True: + expected = self.SUCCESS + elif expected == False: + expected = self.FAILURE + + self.pr('TRYING: ' + message + ', expect ' + expected) + got = None + # If there are stray error messages from a previous operation, + # let's find out now. It can be confusing if we do something illegal + # here and we have multiple messages to sort out. + self.checkStderr() + + # 'yield' runs the subordinate operation, we'll catch any resulting exceptions. + try: + if expected == self.FAILURE: + # Soak up any error messages that happen as a result of the failure. + with self.expectedStderrPattern(r'^.*$', re_flags=re.MULTILINE): + yield + else: + yield + got = self.SUCCESS + except: + got = self.FAILURE + self.cleanStderr() + + message += ' got ' + got + + # If we're about to assert, show some extra info + if expected != got: + message += ': ERROR expected ' + expected + self.checkStderr() + self.pr(message) + self.assertEquals(expected, got) + + # Create a predictable value based on the iteration number and timestamp. + def gen_value(self, iternum, ts): + return str(iternum) + '_' + str(ts) + '_' + 'x' * 1000 + + # Given a number representing an "approximate timestamp", generate a timestamp + # that is near that number, either plus or minus. + def gen_ts(self, approx_ts): + # a number between -10 and 10: + n = self.rand.rand32() % 21 - 10 + ts = approx_ts + n + if ts <= 0: + ts = 1 + return ts + + # Asks whether we should do an illegal operation now. Return yes 5%. + def do_illegal(self): + return self.rand.rand32() % 20 == 0 + + def report(self, func, arg = None): + self.pr('DOING: ' + func + ('' if arg == None else '(' + arg + ')')) + + # Insert a set of rows, each insert in its own transaction, with the + # given timestamps. + def updates(self, value, ds, do_prepare, commit_ts, durable_ts, read_ts): + + # Generate a configuration for a timestamp_transaction() call. + # Returns: 1) whether it expects success, 2) config 3) new running commit timestamp + def timestamp_txn_config(commit_ts, running_commit_ts): + ok = True + config = '' + this_commit_ts = -1 + if self.do_illegal(): + # setting durable timestamp must be after prepare call + config += ',durable_timestamp=' + timestamp_str(self.gen_ts(commit_ts)) + ok = False + + # ODDITY: if we set the durable timestamp (which is illegal at this point), and set a + # valid commit timestamp, the timestamp_transaction() call will fail, but apparently, + # only the durable part fails. The evidence is that the commit timestamp is set, + # as we get a complaint to that effect at the prepare call. The issue is described + # in WT-6995. This seems wrong, and it's hard to work around the problem, so we'll just + # avoid testing that situation for now. Hence the check for a blank configuration. + # When WT-6995 is fixed, remove the "and config = ''" part of the clause immediately + # below, and this entire comment. + if self.rand.rand32() % 2 == 0 and config == '': + if self.do_illegal(): + this_commit_ts = self.oldest_ts - 1 + elif self.do_illegal(): + this_commit_ts = self.stable_ts - 1 + else: + # It's possible this will succeed, we'll check below. + this_commit_ts = self.gen_ts(commit_ts) + config += ',commit_timestamp=' + timestamp_str(this_commit_ts) + + if this_commit_ts >= 0: + if this_commit_ts < running_commit_ts: + ok = False + if this_commit_ts < self.stable_ts: + ok = False + if this_commit_ts < self.oldest_ts: + ok = False + if not ok: + this_commit_ts = -1 + if this_commit_ts >= 0: + running_commit_ts = this_commit_ts + return (ok, config, running_commit_ts) + + session = self.session + needs_rollback = False + prepare_config = None + commit_config = 'commit_timestamp=' + timestamp_str(commit_ts) + tstxn1_config = '' + tstxn2_config = '' + + ok_commit = do_prepare or not self.do_illegal() + ok_prepare = True + ok_tstxn1 = True + ok_tstxn2 = True + + # Occasionally put a durable timestamp on a commit without a prepare, + # that will be an error. + if do_prepare or not ok_commit: + commit_config += ',durable_timestamp=' + timestamp_str(durable_ts) + cursor = session.open_cursor(self.uri) + prepare_ts = self.gen_ts(commit_ts) + prepare_config = 'prepare_timestamp=' + timestamp_str(prepare_ts) + begin_config = '' if read_ts < 0 else 'read_timestamp=' + timestamp_str(read_ts) + + # We might do timestamp_transaction calls either before/after inserting + # values, or both. + do_tstxn1 = (self.rand.rand32() % 10 == 0) + do_tstxn2 = (self.rand.rand32() % 10 == 0) + + # Keep track of the commit timestamp that we'll set through the transaction. + # If it decreases, it will trigger an error. At the final commit_transaction + # operation, we'll use the commit_ts. + running_commit_ts = -1 + first_commit_ts = -1 + + if do_tstxn1: + (ok_tstxn1, tstxn1_config, running_commit_ts) = \ + timestamp_txn_config(commit_ts, running_commit_ts) + if first_commit_ts < 0: + first_commit_ts = running_commit_ts + + if do_tstxn2: + (ok_tstxn2, tstxn2_config, running_commit_ts) = \ + timestamp_txn_config(commit_ts, running_commit_ts) + if first_commit_ts < 0: + first_commit_ts = running_commit_ts + + # WT-7011: + # ODDITY: If any setting of the timestamp fails, then an ASSERT will be hit in prepare. + # Avoid this, it will crash the test suite when diagnostic mode is enabled, and without + # diagnostic mode, the prepare appears to succeed! Comment out the statement marked + # "AVOID ASSERT" below to see it happen. We should fix this to not assert in prepare, + # and either return an error in prepare, or fully succeed (forgiving the previous + # bad timestamp_transaction and allowing subsequent commit). If the former, + # then just remove the "AVOID ASSERT" line below. If the latter, then remove + # the entire if statement enclosing the "AVOID ASSERT". + if not ok_tstxn1 or not ok_tstxn2: + # If a setting of the timestamp fails, the prepare and commit both fail. + ok_prepare = False + ok_commit = False + do_prepare = False # AVOID ASSERT + + if running_commit_ts >= 0 and do_prepare: + # Cannot set prepare timestamp after commit timestamp is successfully set. + ok_prepare = False + + if do_prepare: + if commit_ts < prepare_ts: + ok_commit = False + if prepare_ts < self.oldest_ts: + ok_prepare = False + + # If the final commit is too old, we'll fail. + if commit_ts < self.oldest_ts or commit_ts < self.stable_ts: + ok_commit = False + + # ODDITY: We don't have to move the commit_ts ahead, but it has to be + # at least the value of the first commit timestamp set. + if commit_ts < first_commit_ts: + ok_commit = False + + # If a prepare fails, the commit fails as well. + if not ok_prepare: + ok_commit = False + + msg = 'inserts with commit config(' + commit_config + ')' + + try: + for i in range(0, self.nrows): + needs_rollback = False + if self.do_illegal(): + # Illegal outside of transaction + self.report('prepare_transaction', prepare_config) + with self.expect(False, 'prepare outside of transaction'): + session.prepare_transaction(prepare_config) + + with self.expect(True, 'begin_transaction(' + begin_config + ')'): + session.begin_transaction() + needs_rollback = True + + if do_tstxn1: + with self.expect(ok_tstxn1, 'timestamp_transaction(' + tstxn1_config + ')'): + session.timestamp_transaction(tstxn1_config) + + self.report('set key/value') + with self.expect(True, 'cursor insert'): + cursor[ds.key(i)] = value + + if do_tstxn2: + with self.expect(ok_tstxn2, 'timestamp_transaction(' + tstxn2_config + ')'): + session.timestamp_transaction(tstxn2_config) + + if do_prepare: + self.report('prepare_transaction', prepare_config) + with self.expect(ok_prepare, 'prepare'): + session.prepare_transaction(prepare_config) + + # Doing anything else after the prepare, like a timestamp_transaction(), will fail + # with a WT panic. Don't do that, or else we can't do anything more in this test. + + # If we did a successful prepare and are set up (by virtue of bad timestamps) + # to do a bad commit, WT will panic, and the test cannot continue. + # Only proceed with the commit if we have don't have that particular case. + if ok_commit or not do_prepare or not ok_prepare: + needs_rollback = False + self.report('commit_transaction', commit_config) + with self.expect(ok_commit, 'commit'): + session.commit_transaction(commit_config) + self.commit_value = value + if needs_rollback: + # Rollback this one transaction, and continue the loop + self.report('rollback_transaction') + needs_rollback = False + session.rollback_transaction() + except Exception as e: + # We don't expect any exceptions, they should be caught as part of self.expect statements. + self.pr(msg + 'UNEXPECTED EXCEPTION!') + self.pr(msg + 'fail: ' + str(e)) + raise e + cursor.close() + + def make_timestamp_config(self, oldest, stable, commit, durable): + configs = [] + # Get list of 'oldest_timestamp=value' etc. that have non-negative values. + for ts_name in ['oldest', 'stable', 'commit', 'durable']: + val = eval(ts_name) + if val >= 0: + configs.append(ts_name + '_timestamp=' + timestamp_str(val)) + return ','.join(configs) + + # Determine whether we expect the set_timestamp to succeed. + def expected_result_set_timestamp(self, oldest, stable, commit, durable): + + # Update the current expected value. ts is the timestamp being set. + # If "ts" is negative, ignore it, it's not being set in this call. + # It is unexpected if "ts" is before the "before" timestamp. + # The "before" timestamp could be updated during this call + # with value "before_arg", if not, use the global value for "before". + def expected_newer(expected, ts, before_arg, before_global): + if expected and ts >= 0: + if before_arg >= 0: + if before_arg > ts: + expected = self.FAILURE + else: + if before_global > ts: + expected = self.FAILURE + return expected + + expected = self.SUCCESS + if oldest >= 0 and stable < 0: + expected = expected_newer(expected, self.stable_ts, oldest, self.oldest_ts) + expected = expected_newer(expected, stable, oldest, self.oldest_ts) + expected = expected_newer(expected, commit, oldest, self.oldest_ts) + expected = expected_newer(expected, commit, stable, self.stable_ts) + + # If commit timestamp is set, durable timestamp is ignored. This seems to be + # a temporary situation, see TODO in txn_timestamp.c. + if commit < 0: + expected = expected_newer(expected, durable, oldest, self.oldest_ts) + expected = expected_newer(expected, durable, stable, self.stable_ts) + + return expected + + def set_global_timestamps(self, oldest, stable, commit, durable): + config = self.make_timestamp_config(oldest, stable, commit, durable) + expected = self.expected_result_set_timestamp(oldest, stable, commit, durable) + + with self.expect(expected, 'set_timestamp(' + config + ')'): + self.conn.set_timestamp(config) + + # Predict what we expect to happen to the timestamps. + if expected == self.SUCCESS: + # If that passes, then independently, oldest and stable can advance, but if they + # are less than the current value, that is silently ignored. + if oldest >= self.oldest_ts: + self.oldest_ts = oldest + self.pr('updating oldest: ' + str(oldest)) + if stable >= self.stable_ts: + self.stable_ts = stable + self.pr('updating stable: ' + str(stable)) + + # Make sure the state of global timestamps is what we think. + expect_query_oldest = timestamp_str(self.oldest_ts) + expect_query_stable = timestamp_str(self.stable_ts) + query_oldest = self.conn.query_timestamp('get=oldest') + query_stable = self.conn.query_timestamp('get=stable') + + self.assertEquals(expect_query_oldest, query_oldest) + self.assertEquals(expect_query_stable, query_stable) + self.pr('oldest now: ' + query_oldest) + self.pr('stable now: ' + query_stable) + + if expected == self.FAILURE: + self.cleanStderr() + + def test_timestamp_randomizer(self): + # Local function to generate a random timestamp, or return -1 + def maybe_ts(do_gen, iternum): + if do_gen: + return self.gen_ts(iternum) + else: + return -1 + + if wttest.islongtest(): + iterations = 100000 + else: + iterations = 1000 + + create_params = 'value_format=S,key_format=i' + self.session.create(self.uri, create_params) + + self.set_global_timestamps(1, 1, -1, -1) + + # Create tables with no entries + ds = SimpleDataSet( + self, self.uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + + # We do a bunch of iterations, doing transactions, prepare, and global timestamp calls + # with timestamps that are sometimes valid, sometimes not. We use the iteration number + # as an "approximate timestamp", and generate timestamps for our calls that are near + # that number (within 10). Thus, as the test runs, the timestamps generally get larger. + # We always know the state of global timestamps, so we can predict the success/failure + # on each call. + self.commit_value = '<NOT_SET>' + for iternum in range(1, iterations): + self.pr('\n===== ITERATION ' + str(iternum) + '/' + str(iterations)) + self.pr('RANDOM: ({0},{1})'.format(self.rand.seedw,self.rand.seedz)) + if self.rand.rand32() % 10 != 0: + commit_ts = self.gen_ts(iternum) + durable_ts = self.gen_ts(iternum) + do_prepare = (self.rand.rand32() % 20 == 0) + if self.rand.rand32() % 2 == 0: + read_ts = self.gen_ts(iternum) + else: + read_ts = -1 # no read_timestamp used in txn + if do_prepare: + # If we doing a prepare, we must abide by some additional rules. + # If we don't we'll immediately panic + if commit_ts < self.oldest_ts: + commit_ts = self.oldest_ts + if durable_ts < commit_ts: + durable_ts = commit_ts + value = self.gen_value(iternum, commit_ts) + self.updates(value, ds, do_prepare, commit_ts, durable_ts, read_ts) + + if self.rand.rand32() % 2 == 0: + # Set some combination of the global timestamps + r = self.rand.rand32() % 16 + oldest = maybe_ts((r & 0x1) != 0, iternum) + stable = maybe_ts((r & 0x2) != 0, iternum) + commit = maybe_ts((r & 0x4) != 0, iternum) + durable = maybe_ts((r & 0x8) != 0, iternum) + self.set_global_timestamps(oldest, stable, commit, durable) + + # Make sure the resulting rows are what we expect. + cursor = self.session.open_cursor(self.uri) + expect_key = 0 + expect_value = self.commit_value + for k,v in cursor: + self.assertEquals(k, expect_key) + self.assertEquals(v, expect_value) + expect_key += 1 + + # Although it's theoretically possible to never successfully update a single row, + # with a large number of iterations that should never happen. I'd rather catch + # a test code error where we mistakenly don't update any rows. + self.assertGreater(expect_key, 0) + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 0e7bb3ae6f9..0d11bfb193b 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -113,6 +113,14 @@ class CapturedFd(object): contents + '"') self.expectpos = os.path.getsize(self.filename) + def ignorePreviousOutput(self): + """ + Ignore any output up to this point. + """ + if self.file != None: + self.file.flush() + self.expectpos = os.path.getsize(self.filename) + def checkAdditional(self, testcase, expect): """ Check to see that an additional string has been added to the @@ -127,7 +135,7 @@ class CapturedFd(object): gotstr + '"') self.expectpos = os.path.getsize(self.filename) - def checkAdditionalPattern(self, testcase, pat): + def checkAdditionalPattern(self, testcase, pat, re_flags = 0): """ Check to see that an additional string has been added to the output file. If it has not, raise it as a test failure. @@ -136,7 +144,7 @@ class CapturedFd(object): if self.file != None: self.file.flush() gotstr = self.readFileFrom(self.filename, self.expectpos, 1500) - if re.search(pat, gotstr) == None: + if re.search(pat, gotstr, re_flags) == None: testcase.fail('in ' + self.desc + ', expected pattern "' + pat + '", but got "' + gotstr + '"') @@ -419,6 +427,18 @@ class WiredTigerTestCase(unittest.TestCase): if exc_list and exc_list[-1][0] is self: return exc_list[-1][1] + def cleanStderr(self): + self.captureerr.ignorePreviousOutput() + + def cleanStdout(self): + self.captureout.ignorePreviousOutput() + + def checkStderr(self): + self.captureerr.check(self) + + def checkStdout(self): + self.captureout.check(self) + def tearDown(self): # This approach works for all our support Python versions and # is suggested by one of the answers in: @@ -512,24 +532,24 @@ class WiredTigerTestCase(unittest.TestCase): self.captureerr.checkAdditional(self, expect) @contextmanager - def expectedStdoutPattern(self, pat): + def expectedStdoutPattern(self, pat, re_flags=0): self.captureout.check(self) yield - self.captureout.checkAdditionalPattern(self, pat) + self.captureout.checkAdditionalPattern(self, pat, re_flags) @contextmanager - def expectedStderrPattern(self, pat): + def expectedStderrPattern(self, pat, re_flags=0): self.captureerr.check(self) yield - self.captureerr.checkAdditionalPattern(self, pat) + self.captureerr.checkAdditionalPattern(self, pat, re_flags) - def ignoreStdoutPatternIfExists(self, pat): + def ignoreStdoutPatternIfExists(self, pat, re_flags=0): if self.captureout.hasUnexpectedOutput(self): - self.captureout.checkAdditionalPattern(self, pat) + self.captureout.checkAdditionalPattern(self, pat, re_flags) - def ignoreStderrPatternIfExists(self, pat): + def ignoreStderrPatternIfExists(self, pat, re_flags=0): if self.captureerr.hasUnexpectedOutput(self): - self.captureerr.checkAdditionalPattern(self, pat) + self.captureerr.checkAdditionalPattern(self, pat, re_flags) def assertRaisesWithMessage(self, exceptionType, expr, message): """ |