diff options
| author | Alexey Anisimov <alexey.anisimov@mongodb.com> | 2022-02-21 14:08:15 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-02-21 03:39:19 +0000 |
| commit | c0269ba59ae24b89c0d4a44e7abbcc318f1105e1 (patch) | |
| tree | abf59136bf28abb73af5cffb3f009756b3af9d67 | |
| parent | 215311b673962b7c29b8dc435f95e0d70f8ad438 (diff) | |
| download | mongo-c0269ba59ae24b89c0d4a44e7abbcc318f1105e1.tar.gz | |
Import wiredtiger: ca4ffa5ac30b7ec74eb754c9c56489b6670feda3 from branch mongodb-master
ref: 8edcffb096..ca4ffa5ac3
for: 6.0.0
WT-8170 Reduce complexity of timestamp usage assertion API code
28 files changed, 1009 insertions, 1872 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 6a22c454b9d..56c88f2d814 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -46,7 +46,7 @@ common_runtime_config = [ Config('app_metadata', '', r''' application-owned metadata for this object'''), Config('assert', '', r''' - enable enhanced checking. ''', + enable enhanced timestamp checking with error messages and optional core dump''', type='category', subconfig= [ Config('commit_timestamp', 'none', r''' this option is no longer supported, retained for backward compatibility''', @@ -55,37 +55,36 @@ common_runtime_config = [ this option is no longer supported, retained for backward compatibility''', choices=['always', 'never', 'none'], undoc=True), Config('read_timestamp', 'none', r''' - verify that timestamps should \c always or \c never be used on - reads with this table. Verification should be set to \c none - if mixed read use is allowed''', - choices=['always', 'never', 'none']), + check timestamps are \c always or \c never used on reads with + this table, writing an error message if policy is violated. + If the library was built in diagnostic mode, drop core at the + failing check. Should be set to \c none if mixed read use is + allowed''', choices=['always', 'never', 'none']), Config('write_timestamp', 'off', r''' - verify that commit timestamps are used per the configured - \c write_timestamp_usage option for this table''', + check timestamps are used consistently with the configured + \c write_timestamp_usage option for this table, writing + an error message if policy is violated. If the library was + built in diagnostic mode, drop core at the failing check''', choices=['off', 'on']), ]), Config('verbose', '[]', r''' - enable messages for various events. The choices are \c write_timestamp - which adds verbose messages as described by \c write_timestamp_usage. - Options are given as a list, such as \c "verbose=[write_timestamp]"''', - type='list', choices=['write_timestamp']), + this option is no longer supported, retained for backward compatibility''', + type='list', choices=['write_timestamp'], undoc=True), Config('write_timestamp_usage', 'none', r''' - describe how timestamps are expected to be used on modifications to - the table. This option should be used in conjunction with the - corresponding \c write_timestamp configuration under the \c assert and - \c verbose options to provide logging and assertions for incorrect - timestamp usage. The choices are \c always which ensures a timestamp is - used for every operation on a table, \c ordered which ensures that - once timestamps are used for a key, they are always used, and also - that subsequent updates to each key must use increasing timestamps, + describe how timestamps are expected to be used on modifications + to the table. This option should be used in conjunction with the + corresponding \c write_timestamp configuration under the \c assert + option to provide logging and assertions for incorrect timestamp + usage. The choices are \c always which ensures a timestamp is used + for every operation on a table, \c ordered which ensures that once + timestamps are used for a key, they are always used, and also that + subsequent updates to each key must use increasing timestamps, \c mixed_mode is like \c ordered except that updates with no timestamp - are allowed and have the effect of resetting the chain of updates - once the transaction ID based snapshot is no longer relevant, \c - never enforces that timestamps are never used for a table and \c - none does not enforce any expectation on timestamp usage meaning - that no log message or assertions will be produced regardless of the - corresponding \c assert and \c verbose settings''', choices=['always', - 'mixed_mode', 'never', 'none', 'ordered']), + are allowed at any time, \c never enforces that timestamps are never + used for a table and \c none does not enforce any expectation on + timestamp usage meaning that no log message or assertions will be + produced regardless of the corresponding \c assert setting''', + choices=['always', 'mixed_mode', 'never', 'none', 'ordered']), ] # Metadata shared by all schema objects @@ -607,7 +606,7 @@ connection_runtime_config = [ control the settings of various extended debugging features''', type='category', subconfig=[ Config('corruption_abort', 'true', r''' - if true, dump the core in the diagnostic mode on encountering the data corruption.''', + if true and built in diagnostic mode, dump core in the case of data corruption''', type='boolean'), Config('checkpoint_retention', '0', r''' adjust log removal to retain the log records of this number diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 71fd4ba5931..e3f8a95cc0f 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "8edcffb096846d3cf685f7ab7a41338d767a5c45" + "commit": "ca4ffa5ac30b7ec74eb754c9c56489b6670feda3" } diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 60a567b56a8..a6ff539bce3 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -530,7 +530,7 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_STAT_CONN_DATA_INCR(session, cursor_search); - WT_RET(__wt_txn_search_check(session)); + __wt_txn_search_check(session); __cursor_state_save(cursor, &state); /* @@ -643,7 +643,7 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_STAT_CONN_DATA_INCR(session, cursor_search_near); - WT_RET(__wt_txn_search_check(session)); + __wt_txn_search_check(session); __cursor_state_save(cursor, &state); /* @@ -1820,9 +1820,12 @@ __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_BTREE *btree; WT_DECL_RET; WT_SESSION_IMPL *session; + bool logging; btree = CUR2BT(start); session = CUR2S(start); + logging = __wt_log_op(session); + WT_STAT_DATA_INCR(session, cursor_truncate); WT_RET(__wt_txn_autocommit_check(session)); @@ -1835,7 +1838,7 @@ __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) * We deal with this here by logging the truncate range first, then (in the logging code) * disabling writing of the in-memory remove records to disk. */ - if (FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ENABLED)) + if (logging) WT_RET(__wt_txn_truncate_log(session, start, stop)); switch (btree->type) { @@ -1860,7 +1863,7 @@ __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) } err: - if (FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_ENABLED)) + if (logging) __wt_txn_truncate_end(session); return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index db3fd5f8ab3..4f367422598 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -133,9 +133,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); -#ifdef HAVE_DIAGNOSTIC upd->prev_durable_ts = prev_upd_ts; -#endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -216,9 +214,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U if (upd_arg == NULL) { WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); -#ifdef HAVE_DIAGNOSTIC upd->prev_durable_ts = prev_upd_ts; -#endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -262,7 +258,8 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U /* If the update was successful, add it to the in-memory log. */ if (logged && modify_type != WT_UPDATE_RESERVE) { - WT_ERR(__wt_txn_log_op(session, cbt)); + if (__wt_log_op(session)) + WT_ERR(__wt_txn_log_op(session, cbt)); /* * In case of append, the recno (key) for the value is assigned now. Set the key in the diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index 2968ea3d227..e7b45e61cac 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -114,9 +114,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, /* Allocate a WT_UPDATE structure and transaction ID. */ WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); -#ifdef HAVE_DIAGNOSTIC upd->prev_durable_ts = prev_upd_ts; -#endif WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -253,7 +251,8 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, /* If the update was successful, add it to the in-memory log. */ if (logged && modify_type != WT_UPDATE_RESERVE) { - WT_ERR(__wt_txn_log_op(session, cbt)); + if (__wt_log_op(session)) + WT_ERR(__wt_txn_log_op(session, cbt)); /* * Set the key in the transaction operation to be used in case this transaction is prepared diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index c2c72218872..bfb0be712fd 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -456,50 +456,44 @@ err: } /* - * __conn_dhandle_config_parse -- - * Parse out configuration settings relevant for the data handle. + * __conn_dhandle_config_parse_ts -- + * Parse out timestamp configuration settings for the data handle. */ static int -__conn_dhandle_config_parse(WT_SESSION_IMPL *session) +__conn_dhandle_config_parse_ts(WT_SESSION_IMPL *session) { - WT_CONFIG_ITEM cval, sval; + WT_CONFIG_ITEM cval; WT_DATA_HANDLE *dhandle; - WT_DECL_RET; + uint32_t flags; const char **cfg; dhandle = session->dhandle; + flags = 0; cfg = dhandle->cfg; - /* Clear all the flags. */ - dhandle->ts_flags = 0; - - /* Debugging information */ - WT_RET(__wt_config_gets(session, cfg, "assert.write_timestamp", &cval)); - if (WT_STRING_MATCH("on", cval.str, cval.len)) - FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_WRITE); - + /* Timestamp debugging: asserts. */ WT_RET(__wt_config_gets(session, cfg, "assert.read_timestamp", &cval)); if (WT_STRING_MATCH("always", cval.str, cval.len)) - FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_ALWAYS); + LF_SET(WT_DHANDLE_TS_ASSERT_READ_ALWAYS); else if (WT_STRING_MATCH("never", cval.str, cval.len)) - FLD_SET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_NEVER); - - /* Setup verbose options */ - WT_RET(__wt_config_gets(session, cfg, "verbose", &cval)); - if ((ret = __wt_config_subgets(session, &cval, "write_timestamp", &sval)) == 0 && sval.val != 0) - FLD_SET(dhandle->ts_flags, WT_DHANDLE_VERB_TS_WRITE); - WT_RET_NOTFOUND_OK(ret); + LF_SET(WT_DHANDLE_TS_ASSERT_READ_NEVER); + WT_RET(__wt_config_gets(session, cfg, "assert.write_timestamp", &cval)); + if (WT_STRING_MATCH("on", cval.str, cval.len)) + LF_SET(WT_DHANDLE_TS_ASSERT_WRITE); - /* Setup timestamp usage hints */ + /* Timestamp debugging: write_timestamp_usage. */ WT_RET(__wt_config_gets(session, cfg, "write_timestamp_usage", &cval)); if (WT_STRING_MATCH("always", cval.str, cval.len)) - FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_ALWAYS); + LF_SET(WT_DHANDLE_TS_ALWAYS); else if (WT_STRING_MATCH("mixed_mode", cval.str, cval.len)) - FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_MIXED_MODE); + LF_SET(WT_DHANDLE_TS_MIXED_MODE); else if (WT_STRING_MATCH("never", cval.str, cval.len)) - FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_NEVER); + LF_SET(WT_DHANDLE_TS_NEVER); else if (WT_STRING_MATCH("ordered", cval.str, cval.len)) - FLD_SET(dhandle->ts_flags, WT_DHANDLE_TS_ORDERED); + LF_SET(WT_DHANDLE_TS_ORDERED); + + /* Reset the flags. */ + dhandle->ts_flags = flags; return (0); } @@ -542,7 +536,7 @@ __wt_conn_dhandle_open(WT_SESSION_IMPL *session, const char *cfg[], uint32_t fla /* Discard any previous configuration, set up the new configuration. */ __conn_dhandle_config_clear(session); WT_ERR(__conn_dhandle_config_set(session)); - WT_ERR(__conn_dhandle_config_parse(session)); + WT_ERR(__conn_dhandle_config_parse_ts(session)); switch (dhandle->type) { case WT_DHANDLE_TYPE_BTREE: diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 892972e28e4..14b26fbfe14 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -104,7 +104,6 @@ do { \ bool __autotxn = false, __update = false; \ API_CALL(s, h, n, dh, config, cfg); \ - __wt_txn_timestamp_flags(s); \ __autotxn = !F_ISSET((s)->txn, WT_TXN_AUTOCOMMIT | WT_TXN_RUNNING); \ if (__autotxn) \ F_SET((s)->txn, WT_TXN_AUTOCOMMIT); \ @@ -117,7 +116,6 @@ do { \ bool __autotxn = false, __update = false; \ API_CALL_NOCONF(s, h, n, dh); \ - __wt_txn_timestamp_flags(s); \ __autotxn = !F_ISSET((s)->txn, WT_TXN_AUTOCOMMIT | WT_TXN_RUNNING); \ if (__autotxn) \ F_SET((s)->txn, WT_TXN_AUTOCOMMIT); \ diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index a6589a5b944..5f28e3cc540 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -1126,22 +1126,25 @@ struct __wt_ikey { /* * WT_UPDATE -- - * Entries on leaf pages can be updated, either modified or deleted. - * Updates to entries referenced from the WT_ROW and WT_COL arrays are - * stored in the page's WT_UPDATE array. When the first element on a page - * is updated, the WT_UPDATE array is allocated, with one slot for every - * existing element in the page. A slot points to a WT_UPDATE structure; - * if more than one update is done for an entry, WT_UPDATE structures are - * formed into a forward-linked list. + * + * Entries on leaf pages can be updated, either modified or deleted. Updates to entries in the + * WT_ROW and WT_COL arrays are stored in the page's WT_UPDATE array. When the first element on a + * page is updated, the WT_UPDATE array is allocated, with one slot for every existing element in + * the page. A slot points to a WT_UPDATE structure; if more than one update is done for an entry, + * WT_UPDATE structures are formed into a forward-linked list. */ struct __wt_update { volatile uint64_t txnid; /* transaction ID */ wt_timestamp_t durable_ts; /* timestamps */ wt_timestamp_t start_ts; -#ifdef HAVE_DIAGNOSTIC + + /* + * The durable timestamp of the previous update in the update chain. This timestamp is used for + * diagnostic checks only, and could be removed to reduce the size of the structure should that + * be necessary. + */ wt_timestamp_t prev_durable_ts; -#endif WT_UPDATE *next; /* forward-linked list */ @@ -1186,11 +1189,7 @@ struct __wt_update { * WT_UPDATE_SIZE is the expected structure size excluding the payload data -- we verify the build * to ensure the compiler hasn't inserted padding. */ -#ifdef HAVE_DIAGNOSTIC #define WT_UPDATE_SIZE 47 -#else -#define WT_UPDATE_SIZE 39 -#endif /* * The memory size of an update: include some padding because this is such a common case that diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index c649f747446..ad967876010 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -129,14 +129,13 @@ struct __wt_data_handle { uint32_t flags; /* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_DHANDLE_ASSERT_TS_READ_ALWAYS 0x01u /* Assert read always checking. */ -#define WT_DHANDLE_ASSERT_TS_READ_NEVER 0x02u /* Assert read never checking. */ -#define WT_DHANDLE_ASSERT_TS_WRITE 0x04u /* Assert write checking. */ -#define WT_DHANDLE_TS_ALWAYS 0x08u /* Handle using always checking. */ +#define WT_DHANDLE_TS_ALWAYS 0x01u /* Handle using always checking. */ +#define WT_DHANDLE_TS_ASSERT_READ_ALWAYS 0x02u /* Assert read always checking. */ +#define WT_DHANDLE_TS_ASSERT_READ_NEVER 0x04u /* Assert read never checking. */ +#define WT_DHANDLE_TS_ASSERT_WRITE 0x08u /* Assert write checking. */ #define WT_DHANDLE_TS_MIXED_MODE 0x10u /* Handle using mixed mode timestamps checking. */ #define WT_DHANDLE_TS_NEVER 0x20u /* Handle never using timestamps checking. */ #define WT_DHANDLE_TS_ORDERED 0x40u /* Handle using ordered timestamps checking. */ -#define WT_DHANDLE_VERB_TS_WRITE 0x80u /* Handle verbose logging for timestamps usage. */ /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t ts_flags; }; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index c1d0b8ddb05..3cf657c87f9 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1621,8 +1621,6 @@ extern int __wt_verbose_dump_txn(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_verbose_dump_txn_one(WT_SESSION_IMPL *session, WT_SESSION_IMPL *txn_session, int error_code, const char *error_string) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_verbose_dump_update(WT_SESSION_IMPL *session, WT_UPDATE *upd) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_verify_ckpt_load(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_BLOCK_CKPT *ci) @@ -1926,6 +1924,8 @@ static inline bool __wt_isprint(u_char c) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unuse static inline bool __wt_isspace(u_char c) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_leaf_page_can_split(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline bool __wt_log_op(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_off_page(WT_PAGE *page, const void *p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_op_timer_fired(WT_SESSION_IMPL *session) @@ -2161,8 +2161,6 @@ static inline int __wt_txn_read_upd_list(WT_SESSION_IMPL *session, WT_CURSOR_BTR static inline int __wt_txn_read_upd_list_internal(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd, WT_UPDATE **prepare_updp, WT_UPDATE **restored_updp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_txn_search_check(WT_SESSION_IMPL *session) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, u_int modify_type, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp, @@ -2346,7 +2344,7 @@ static inline void __wt_txn_op_set_recno(WT_SESSION_IMPL *session, uint64_t recn static inline void __wt_txn_op_set_timestamp(WT_SESSION_IMPL *session, WT_TXN_OP *op); static inline void __wt_txn_pinned_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t *pinned_tsp); static inline void __wt_txn_read_last(WT_SESSION_IMPL *session); -static inline void __wt_txn_timestamp_flags(WT_SESSION_IMPL *session); +static inline void __wt_txn_search_check(WT_SESSION_IMPL *session); static inline void __wt_txn_unmodify(WT_SESSION_IMPL *session); static inline void __wt_upd_value_assign(WT_UPDATE_VALUE *upd_value, WT_UPDATE *upd); static inline void __wt_upd_value_clear(WT_UPDATE_VALUE *upd_value); diff --git a/src/third_party/wiredtiger/src/include/log_inline.h b/src/third_party/wiredtiger/src/include/log_inline.h index 861d0afb022..22eef9dc9a0 100644 --- a/src/third_party/wiredtiger/src/include/log_inline.h +++ b/src/third_party/wiredtiger/src/include/log_inline.h @@ -24,3 +24,31 @@ __wt_log_cmp(WT_LSN *lsn1, WT_LSN *lsn2) return (l1 < l2 ? -1 : (l1 > l2 ? 1 : 0)); } + +/* + * __wt_log_op -- + * Return if an operation should be logged. + */ +static inline bool +__wt_log_op(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* + * Objects with checkpoint durability don't need logging unless we're in debug mode. That rules + * out almost all log records, check it first. + */ + if (F_ISSET(S2BT(session), WT_BTREE_NO_LOGGING) && + !FLD_ISSET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE)) + return (false); + + /* Logging must be enabled, and outside of recovery. */ + if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) + return (false); + if (F_ISSET(conn, WT_CONN_RECOVERING)) + return (false); + + return (true); +} diff --git a/src/third_party/wiredtiger/src/include/serial_inline.h b/src/third_party/wiredtiger/src/include/serial_inline.h index 4f6384732dc..8380c3d5b89 100644 --- a/src/third_party/wiredtiger/src/include/serial_inline.h +++ b/src/third_party/wiredtiger/src/include/serial_inline.h @@ -230,9 +230,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page *updp = NULL; prev_upd_ts = WT_TS_NONE; -#ifdef HAVE_DIAGNOSTIC prev_upd_ts = upd->prev_durable_ts; -#endif /* * All structure setup must be flushed before the structure is entered into the list. We need a @@ -248,9 +246,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page return (ret); } } -#ifdef HAVE_DIAGNOSTIC upd->prev_durable_ts = prev_upd_ts; -#endif /* * Increment in-memory footprint after swapping the update into place. Safe because the diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index a51284d4a0f..96b72f10e43 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -203,13 +203,12 @@ struct __wt_session_impl { #define WT_SESSION_INTERNAL 0x00200u #define WT_SESSION_LOGGING_INMEM 0x00400u #define WT_SESSION_NO_DATA_HANDLES 0x00800u -#define WT_SESSION_NO_LOGGING 0x01000u -#define WT_SESSION_NO_RECONCILE 0x02000u -#define WT_SESSION_QUIET_CORRUPT_FILE 0x04000u -#define WT_SESSION_READ_WONT_NEED 0x08000u -#define WT_SESSION_RESOLVING_TXN 0x10000u -#define WT_SESSION_ROLLBACK_TO_STABLE 0x20000u -#define WT_SESSION_SCHEMA_TXN 0x40000u +#define WT_SESSION_NO_RECONCILE 0x01000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x02000u +#define WT_SESSION_READ_WONT_NEED 0x04000u +#define WT_SESSION_RESOLVING_TXN 0x08000u +#define WT_SESSION_ROLLBACK_TO_STABLE 0x10000u +#define WT_SESSION_SCHEMA_TXN 0x20000u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 8f222bcadbb..7c60dfc23bb 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -242,6 +242,8 @@ struct __wt_txn_op { uint32_t flags; }; +#define WT_TS_VERBOSE_PREFIX "unexpected timestamp usage: " + /* * WT_TXN -- * Per-session transaction context. @@ -323,29 +325,24 @@ struct __wt_txn { */ /* AUTOMATIC FLAG VALUE GENERATION START 0 */ -#define WT_TXN_AUTOCOMMIT 0x000001u -#define WT_TXN_ERROR 0x000002u -#define WT_TXN_HAS_ID 0x000004u -#define WT_TXN_HAS_SNAPSHOT 0x000008u -#define WT_TXN_HAS_TS_COMMIT 0x000010u -#define WT_TXN_HAS_TS_DURABLE 0x000020u -#define WT_TXN_HAS_TS_PREPARE 0x000040u -#define WT_TXN_IGNORE_PREPARE 0x000080u -#define WT_TXN_PREPARE 0x000100u -#define WT_TXN_PREPARE_IGNORE_API_CHECK 0x000200u -#define WT_TXN_READONLY 0x000400u -#define WT_TXN_RUNNING 0x000800u -#define WT_TXN_SHARED_TS_DURABLE 0x001000u -#define WT_TXN_SHARED_TS_READ 0x002000u -#define WT_TXN_SYNC_SET 0x004000u -#define WT_TXN_TS_ROUND_PREPARED 0x008000u -#define WT_TXN_TS_ROUND_READ 0x010000u -#define WT_TXN_TS_WRITE_ALWAYS 0x020000u -#define WT_TXN_TS_WRITE_MIXED_MODE 0x040000u -#define WT_TXN_TS_WRITE_NEVER 0x080000u -#define WT_TXN_TS_WRITE_ORDERED 0x100000u -#define WT_TXN_UPDATE 0x200000u -#define WT_TXN_VERB_TS_WRITE 0x400000u +#define WT_TXN_AUTOCOMMIT 0x00001u +#define WT_TXN_ERROR 0x00002u +#define WT_TXN_HAS_ID 0x00004u +#define WT_TXN_HAS_SNAPSHOT 0x00008u +#define WT_TXN_HAS_TS_COMMIT 0x00010u +#define WT_TXN_HAS_TS_DURABLE 0x00020u +#define WT_TXN_HAS_TS_PREPARE 0x00040u +#define WT_TXN_IGNORE_PREPARE 0x00080u +#define WT_TXN_PREPARE 0x00100u +#define WT_TXN_PREPARE_IGNORE_API_CHECK 0x00200u +#define WT_TXN_READONLY 0x00400u +#define WT_TXN_RUNNING 0x00800u +#define WT_TXN_SHARED_TS_DURABLE 0x01000u +#define WT_TXN_SHARED_TS_READ 0x02000u +#define WT_TXN_SYNC_SET 0x04000u +#define WT_TXN_TS_ROUND_PREPARED 0x08000u +#define WT_TXN_TS_ROUND_READ 0x10000u +#define WT_TXN_UPDATE 0x20000u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index 3f84a80db8b..87400605fb5 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -66,40 +66,6 @@ __wt_txn_err_set(WT_SESSION_IMPL *session, int ret) } /* - * __wt_txn_timestamp_flags -- - * Set transaction related timestamp flags. - */ -static inline void -__wt_txn_timestamp_flags(WT_SESSION_IMPL *session) -{ - WT_BTREE *btree; - WT_DATA_HANDLE *dhandle; - - dhandle = session->dhandle; - if (dhandle == NULL) - return; - btree = S2BT(session); - if (btree == NULL) - return; - - if (!FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_WRITE)) - return; - - if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_ALWAYS)) - F_SET(session->txn, WT_TXN_TS_WRITE_ALWAYS); - if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_MIXED_MODE)) - F_SET(session->txn, WT_TXN_TS_WRITE_MIXED_MODE); - if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_NEVER)) - F_SET(session->txn, WT_TXN_TS_WRITE_NEVER); - if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_TS_ORDERED)) - F_SET(session->txn, WT_TXN_TS_WRITE_ORDERED); - - /* Remember if any type of verbose tracking is encountered by the transaction. */ - if (FLD_ISSET(dhandle->ts_flags, WT_DHANDLE_VERB_TS_WRITE)) - F_SET(session->txn, WT_TXN_VERB_TS_WRITE); -} - -/* * __wt_txn_op_set_recno -- * Set the latest transaction operation with the given recno. */ @@ -439,7 +405,8 @@ __wt_txn_modify_page_delete(WT_SESSION_IMPL *session, WT_REF *ref) ref->ft_info.del->txnid = txn->id; __wt_txn_op_set_timestamp(session, op); - WT_ERR(__wt_txn_log_op(session, NULL)); + if (__wt_log_op(session)) + WT_ERR(__wt_txn_log_op(session, NULL)); return (0); err: @@ -1297,30 +1264,43 @@ __wt_txn_id_check(WT_SESSION_IMPL *session) /* * __wt_txn_search_check -- - * Check if the current transaction can search. + * Check if a search by the current transaction violates timestamp rules. */ -static inline int +static inline void __wt_txn_search_check(WT_SESSION_IMPL *session) { - WT_BTREE *btree; WT_TXN *txn; + uint32_t flags; + const char *name; - btree = S2BT(session); txn = session->txn; + flags = session->dhandle->ts_flags; + name = session->dhandle->name; - /* - * If the user says a table should always use a read timestamp, verify this transaction has one. - * Same if it should never have a read timestamp. - */ - if (!F_ISSET(S2C(session), WT_CONN_RECOVERING) && - FLD_ISSET(btree->dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_ALWAYS) && - !F_ISSET(txn, WT_TXN_SHARED_TS_READ)) - WT_RET_MSG(session, EINVAL, "read_timestamp required and none set on this transaction"); - if (FLD_ISSET(btree->dhandle->ts_flags, WT_DHANDLE_ASSERT_TS_READ_NEVER) && - F_ISSET(txn, WT_TXN_SHARED_TS_READ)) - WT_RET_MSG( - session, EINVAL, "no read_timestamp required and timestamp set on this transaction"); - return (0); + /* Timestamps are ignored on logged files. */ + if (!F_ISSET(S2C(session), WT_CONN_IN_MEMORY) && !F_ISSET(S2BT(session), WT_BTREE_NO_LOGGING)) + return; + + /* Skip during recovery. */ + if (F_ISSET(S2C(session), WT_CONN_RECOVERING)) + return; + + /* Verify if the table should always or never use a read timestamp. */ + if (LF_ISSET(WT_DHANDLE_TS_ASSERT_READ_ALWAYS) && !F_ISSET(txn, WT_TXN_SHARED_TS_READ)) { + __wt_err(session, EINVAL, + "%s: " WT_TS_VERBOSE_PREFIX "read timestamps required and none set", name); +#ifdef HAVE_DIAGNOSTIC + __wt_abort(session); +#endif + } + + if (LF_ISSET(WT_DHANDLE_TS_ASSERT_READ_NEVER) && F_ISSET(txn, WT_TXN_SHARED_TS_READ)) { + __wt_err(session, EINVAL, + "%s: " WT_TS_VERBOSE_PREFIX "read timestamps disallowed and one set", name); +#ifdef HAVE_DIAGNOSTIC + __wt_abort(session); +#endif + } } /* diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index fbe3c8033e5..55ce391129c 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1042,15 +1042,18 @@ struct __wt_session { * "sequential"; default \c none.} * @config{app_metadata, application-owned metadata for this object., a string; default * empty.} - * @config{assert = (, enable enhanced checking., a set of related configuration options - * defined below.} - * @config{ read_timestamp, verify that timestamps - * should \c always or \c never be used on reads with this table. Verification should be - * set to \c none if mixed read use is allowed., a string\, chosen from the following - * options: \c "always"\, \c "never"\, \c "none"; default \c none.} - * @config{ write_timestamp, verify that commit timestamps are used - * per the configured \c write_timestamp_usage option for this table., a string\, chosen - * from the following options: \c "off"\, \c "on"; default \c off.} + * @config{assert = (, enable enhanced timestamp checking with error messages and optional + * core dump., a set of related configuration options defined below.} + * @config{ read_timestamp, check timestamps are \c always or \c + * never used on reads with this table\, writing an error message if policy is violated. If + * the library was built in diagnostic mode\, drop core at the failing check. Should be set + * to \c none if mixed read use is allowed., a string\, chosen from the following options: + * \c "always"\, \c "never"\, \c "none"; default \c none.} + * @config{ + * write_timestamp, check timestamps are used consistently with the configured \c + * write_timestamp_usage option for this table\, writing an error message if policy is + * violated. If the library was built in diagnostic mode\, drop core at the failing check., + * a string\, chosen from the following options: \c "off"\, \c "on"; default \c off.} * @config{ ),,} * @config{cache_resident, do not ever evict the object's pages from cache. Not compatible * with LSM tables; see @ref tuning_cache_resident for more information., a boolean flag; @@ -1070,24 +1073,18 @@ struct __wt_session { * 0.} * @config{readonly, the file is read-only. All methods that may modify a file are * disabled. See @ref readonly for more information., a boolean flag; default \c false.} - * @config{verbose, enable messages for various events. The choices are \c write_timestamp - * which adds verbose messages as described by \c write_timestamp_usage. Options are given - * as a list\, such as \c "verbose=[write_timestamp]"., a list\, with values chosen from the - * following options: \c "write_timestamp"; default \c [].} * @config{write_timestamp_usage, describe how timestamps are expected to be used on * modifications to the table. This option should be used in conjunction with the - * corresponding \c write_timestamp configuration under the \c assert and \c verbose options - * to provide logging and assertions for incorrect timestamp usage. The choices are \c - * always which ensures a timestamp is used for every operation on a table\, \c ordered - * which ensures that once timestamps are used for a key\, they are always used\, and also - * that subsequent updates to each key must use increasing timestamps\, \c mixed_mode is - * like \c ordered except that updates with no timestamp are allowed and have the effect of - * resetting the chain of updates once the transaction ID based snapshot is no longer - * relevant\, \c never enforces that timestamps are never used for a table and \c none does - * not enforce any expectation on timestamp usage meaning that no log message or assertions - * will be produced regardless of the corresponding \c assert and \c verbose settings., a - * string\, chosen from the following options: \c "always"\, \c "mixed_mode"\, \c "never"\, - * \c "none"\, \c "ordered"; default \c none.} + * corresponding \c write_timestamp configuration under the \c assert option to provide + * logging and assertions for incorrect timestamp usage. The choices are \c always which + * ensures a timestamp is used for every operation on a table\, \c ordered which ensures + * that once timestamps are used for a key\, they are always used\, and also that subsequent + * updates to each key must use increasing timestamps\, \c mixed_mode is like \c ordered + * except that updates with no timestamp are allowed at any time\, \c never enforces that + * timestamps are never used for a table and \c none does not enforce any expectation on + * timestamp usage meaning that no log message or assertions will be produced regardless of + * the corresponding \c assert setting., a string\, chosen from the following options: \c + * "always"\, \c "mixed_mode"\, \c "never"\, \c "none"\, \c "ordered"; default \c none.} * @configend * @errors */ @@ -1118,15 +1115,18 @@ struct __wt_session { * an integer between 512B and 128MB; default \c 4KB.} * @config{app_metadata, application-owned metadata for this object., a string; default * empty.} - * @config{assert = (, enable enhanced checking., a set of related configuration options - * defined below.} - * @config{ read_timestamp, verify that timestamps - * should \c always or \c never be used on reads with this table. Verification should be - * set to \c none if mixed read use is allowed., a string\, chosen from the following - * options: \c "always"\, \c "never"\, \c "none"; default \c none.} - * @config{ write_timestamp, verify that commit timestamps are used - * per the configured \c write_timestamp_usage option for this table., a string\, chosen - * from the following options: \c "off"\, \c "on"; default \c off.} + * @config{assert = (, enable enhanced timestamp checking with error messages and optional + * core dump., a set of related configuration options defined below.} + * @config{ read_timestamp, check timestamps are \c always or \c + * never used on reads with this table\, writing an error message if policy is violated. If + * the library was built in diagnostic mode\, drop core at the failing check. Should be set + * to \c none if mixed read use is allowed., a string\, chosen from the following options: + * \c "always"\, \c "never"\, \c "none"; default \c none.} + * @config{ + * write_timestamp, check timestamps are used consistently with the configured \c + * write_timestamp_usage option for this table\, writing an error message if policy is + * violated. If the library was built in diagnostic mode\, drop core at the failing check., + * a string\, chosen from the following options: \c "off"\, \c "on"; default \c off.} * @config{ ),,} * @config{block_allocation, configure block allocation. Permitted values are \c "best" or * \c "first"; the \c "best" configuration uses a best-fit algorithm\, the \c "first" @@ -1355,24 +1355,18 @@ struct __wt_session { * applications use a WT_ITEM structure to manipulate raw byte arrays. Value items of type * 't' are bitfields\, and when configured with record number type keys\, will be stored * using a fixed-length store., a format string; default \c u.} - * @config{verbose, enable messages for various events. The choices are \c write_timestamp - * which adds verbose messages as described by \c write_timestamp_usage. Options are given - * as a list\, such as \c "verbose=[write_timestamp]"., a list\, with values chosen from the - * following options: \c "write_timestamp"; default \c [].} * @config{write_timestamp_usage, describe how timestamps are expected to be used on * modifications to the table. This option should be used in conjunction with the - * corresponding \c write_timestamp configuration under the \c assert and \c verbose options - * to provide logging and assertions for incorrect timestamp usage. The choices are \c - * always which ensures a timestamp is used for every operation on a table\, \c ordered - * which ensures that once timestamps are used for a key\, they are always used\, and also - * that subsequent updates to each key must use increasing timestamps\, \c mixed_mode is - * like \c ordered except that updates with no timestamp are allowed and have the effect of - * resetting the chain of updates once the transaction ID based snapshot is no longer - * relevant\, \c never enforces that timestamps are never used for a table and \c none does - * not enforce any expectation on timestamp usage meaning that no log message or assertions - * will be produced regardless of the corresponding \c assert and \c verbose settings., a - * string\, chosen from the following options: \c "always"\, \c "mixed_mode"\, \c "never"\, - * \c "none"\, \c "ordered"; default \c none.} + * corresponding \c write_timestamp configuration under the \c assert option to provide + * logging and assertions for incorrect timestamp usage. The choices are \c always which + * ensures a timestamp is used for every operation on a table\, \c ordered which ensures + * that once timestamps are used for a key\, they are always used\, and also that subsequent + * updates to each key must use increasing timestamps\, \c mixed_mode is like \c ordered + * except that updates with no timestamp are allowed at any time\, \c never enforces that + * timestamps are never used for a table and \c none does not enforce any expectation on + * timestamp usage meaning that no log message or assertions will be produced regardless of + * the corresponding \c assert setting., a string\, chosen from the following options: \c + * "always"\, \c "mixed_mode"\, \c "never"\, \c "none"\, \c "ordered"; default \c none.} * @configend * @errors */ @@ -2152,8 +2146,8 @@ struct __wt_connection { * checkpoint_retention, adjust log removal to retain the log records of this number of * checkpoints. Zero or one means perform normal removal., an integer between 0 and 1024; * default \c 0.} - * @config{ corruption_abort, if true\, dump the core - * in the diagnostic mode on encountering the data corruption., a boolean flag; default \c + * @config{ corruption_abort, if true and built in + * diagnostic mode\, dump core in the case of data corruption., a boolean flag; default \c * true.} * @config{ cursor_copy, if true\, use the system allocator to * make a copy of any data returned by a cursor operation and return the copy instead. The @@ -2873,8 +2867,8 @@ struct __wt_connection { * @config{ * checkpoint_retention, adjust log removal to retain the log records of this number of checkpoints. * Zero or one means perform normal removal., an integer between 0 and 1024; default \c 0.} - * @config{ corruption_abort, if true\, dump the core in the diagnostic mode - * on encountering the data corruption., a boolean flag; default \c true.} + * @config{ corruption_abort, if true and built in diagnostic mode\, dump + * core in the case of data corruption., a boolean flag; default \c true.} * @config{ cursor_copy, if true\, use the system allocator to make a copy of * any data returned by a cursor operation and return the copy instead. The copy is freed on the * next cursor operation. This allows memory sanitizers to detect inappropriate references to diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 2b22c3a7cd9..0359054a7c3 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -828,111 +828,94 @@ done: } /* - * __txn_commit_timestamps_usage_check -- - * Print warning messages when encountering unexpected timestamp usage. + * __txn_timestamp_usage_check -- + * Check if a commit will violate timestamp rules. */ -static inline int -__txn_commit_timestamps_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_UPDATE *upd) +static inline void +__txn_timestamp_usage_check(WT_SESSION_IMPL *session, WT_TXN_OP *op, WT_UPDATE *upd) { + WT_BTREE *btree; WT_TXN *txn; wt_timestamp_t op_ts, prev_op_durable_ts; - uint32_t ts_flags; + uint32_t flags; char ts_string[2][WT_TS_INT_STRING_SIZE]; + const char *name; bool txn_has_ts; - /* - * Do not check for timestamp usage in recovery as it is possible that timestamps may be out of - * order due to WiredTiger log replay in recovery doesn't use any timestamps. - */ - if (F_ISSET(S2C(session), WT_CONN_RECOVERING)) - return (0); - + btree = op->btree; txn = session->txn; + flags = btree->dhandle->ts_flags; + name = btree->dhandle->name; txn_has_ts = F_ISSET(txn, WT_TXN_HAS_TS_COMMIT | WT_TXN_HAS_TS_DURABLE); -#define WT_COMMIT_TS_VERB_PREFIX "Commit timestamp unexpected usage: " + /* Skip timestamp usage checks unless both assert and usage configurations are set. */ + if (!LF_ISSET(WT_DHANDLE_TS_ASSERT_WRITE)) + return; + if (!LF_ISSET(WT_DHANDLE_TS_ALWAYS | WT_DHANDLE_TS_MIXED_MODE | WT_DHANDLE_TS_NEVER | + WT_DHANDLE_TS_ORDERED)) + return; - /* If this transaction did not touch any table configured for verbose logging, we're done. */ - if (!F_ISSET(txn, WT_TXN_VERB_TS_WRITE)) - return (0); + /* Timestamps are ignored on logged files. */ + if (!F_ISSET(S2C(session), WT_CONN_IN_MEMORY) && !F_ISSET(btree, WT_BTREE_NO_LOGGING)) + return; - op_ts = upd->start_ts != WT_TS_NONE ? upd->start_ts : txn->commit_timestamp; - ts_flags = op->btree->dhandle->ts_flags; + /* + * Do not check for timestamp usage in recovery. We don't expect recovery to be using timestamps + * when applying commits, and it is possible that timestamps may be out of order in log replay. + */ + if (F_ISSET(S2C(session), WT_CONN_RECOVERING)) + return; - if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ALWAYS) && !txn_has_ts) - __wt_verbose_notice(session, WT_VERB_TRANSACTION, "%s", - WT_COMMIT_TS_VERB_PREFIX - "commit timestamp not used on table configured to require timestamps"); + /* Check for required timestamps. */ + if (LF_ISSET(WT_DHANDLE_TS_ALWAYS) && !txn_has_ts && txn->mod_count != 0) { + __wt_err(session, EINVAL, + "%s: " WT_TS_VERBOSE_PREFIX "timestamp required by table configuration and none set", + name); +#ifdef HAVE_DIAGNOSTIC + __wt_abort(session); +#endif + } - if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_NEVER) && txn_has_ts) - __wt_verbose_notice(session, WT_VERB_TRANSACTION, - WT_COMMIT_TS_VERB_PREFIX - "commit timestamp %s used on table configured to not use timestamps", - __wt_timestamp_to_string(op_ts, ts_string[0])); + op_ts = upd->start_ts != WT_TS_NONE ? upd->start_ts : txn->commit_timestamp; + /* Check for disallowed timestamps. */ + if (LF_ISSET(WT_DHANDLE_TS_NEVER) && txn_has_ts) { + __wt_err(session, EINVAL, + "%s: " WT_TS_VERBOSE_PREFIX "timestamp %s set when disallowed by table configuration", + name, __wt_timestamp_to_string(op_ts, ts_string[0])); #ifdef HAVE_DIAGNOSTIC - prev_op_durable_ts = upd->prev_durable_ts; - - /* - * Exit abnormally as the key consistency mode dictates all updates must use timestamps once - * they have been used. - */ - if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && prev_op_durable_ts != WT_TS_NONE && - !txn_has_ts) { - __wt_verbose_error(session, WT_VERB_TRANSACTION, "%s", - WT_COMMIT_TS_VERB_PREFIX - "no timestamp provided for an update to a " - "table configured to always use timestamps once they are first used"); - WT_ASSERT(session, false); + __wt_abort(session); +#endif } - /* - * Exit abnormally as we don't allow out of order timestamps on a table configured for strict - * ordering. - */ - if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && txn_has_ts && prev_op_durable_ts > op_ts) { - __wt_verbose_error(session, WT_VERB_TRANSACTION, - WT_COMMIT_TS_VERB_PREFIX - "committing a transaction that updates a " - "value with an older timestamp (%s) than is associated with the previous " - "update (%s) on a table configured for strict ordering", - __wt_timestamp_to_string(op_ts, ts_string[0]), - __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1])); - WT_ASSERT(session, false); - } + prev_op_durable_ts = upd->prev_durable_ts; - /* - * Exit abnormally as we don't allow an update without a timestamp if the previous update had an - * associated timestamp. This applies to both tables configured for strict and mixed mode - * orderings. - */ - if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_ORDERED) && prev_op_durable_ts != WT_TS_NONE && - !txn_has_ts) { - __wt_verbose_error(session, WT_VERB_TRANSACTION, - WT_COMMIT_TS_VERB_PREFIX - "committing a transaction that updates a value without " - "a timestamp while the previous update (%s) is timestamped " - "on a table configured for strict ordering", - __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1])); - WT_ASSERT(session, false); + /* Ordered key consistency requires all updates use timestamps, once they are first used. */ + if (LF_ISSET(WT_DHANDLE_TS_ORDERED) && !txn_has_ts && prev_op_durable_ts != WT_TS_NONE) { + __wt_err(session, EINVAL, + "%s: " WT_TS_VERBOSE_PREFIX + "no timestamp provided for an update to a table configured to always use timestamps " + "once they are first used", + name); +#ifdef HAVE_DIAGNOSTIC + __wt_abort(session); +#endif } - if (FLD_ISSET(ts_flags, WT_DHANDLE_TS_MIXED_MODE) && F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && - op_ts != WT_TS_NONE && prev_op_durable_ts > op_ts) { - __wt_verbose_error(session, WT_VERB_TRANSACTION, - WT_COMMIT_TS_VERB_PREFIX - "committing a transaction that updates a " - "value with an older timestamp (%s) than is associated with the previous " - "update (%s) on a table configured for mixed mode ordering", - __wt_timestamp_to_string(op_ts, ts_string[0]), - __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1])); - WT_ASSERT(session, false); - } -#else - WT_UNUSED(prev_op_durable_ts); + /* Ordered and mixed-mode consistency requires all updates be in timestamp order. */ + if (LF_ISSET(WT_DHANDLE_TS_MIXED_MODE | WT_DHANDLE_TS_ORDERED) && txn_has_ts && + prev_op_durable_ts > op_ts) { + __wt_err(session, EINVAL, + "%s: " WT_TS_VERBOSE_PREFIX + "committing a transaction that updates a value with an older timestamp %s than is " + "associated with the previous update %s on a table configured for %s", + name, __wt_timestamp_to_string(op_ts, ts_string[0]), + __wt_timestamp_to_string(prev_op_durable_ts, ts_string[1]), + LF_ISSET(WT_DHANDLE_TS_MIXED_MODE) ? "mixed mode" : "strict ordering"); +#ifdef HAVE_DIAGNOSTIC + __wt_abort(session); #endif - - return (0); + } } /* @@ -1219,7 +1202,7 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, /* A prepared operation that is rolled back will not have a timestamp worth asserting on. */ if (commit) - WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); + __txn_timestamp_usage_check(session, op, upd); for (first_committed_upd = upd; first_committed_upd != NULL && (first_committed_upd->txnid == WT_TXN_ABORTED || @@ -1411,169 +1394,6 @@ err: return (ret); } -#ifdef WT_STANDALONE_BUILD -/* - * __txn_commit_timestamps_assert_standalone -- - * Validate that timestamps provided to commit are legal. - */ -static inline int -__txn_commit_timestamps_assert_standalone(WT_SESSION_IMPL *session, WT_TXN *txn) -{ - WT_CURSOR *cursor; - WT_DECL_RET; - WT_TXN_OP *op; - WT_UPDATE *upd; -#ifdef HAVE_DIAGNOSTIC - wt_timestamp_t op_ts; -#endif - wt_timestamp_t prev_op_durable_ts, prev_op_ts; - u_int i; - bool op_zero_ts, upd_zero_ts; - - cursor = NULL; - - /* - * Error on any valid update structures for the same key that are at a later timestamp or use - * timestamps inconsistently. - */ - for (i = 0, op = txn->mod; i < txn->mod_count; i++, op++) { - switch (op->type) { - case WT_TXN_OP_BASIC_COL: - case WT_TXN_OP_INMEM_COL: - case WT_TXN_OP_BASIC_ROW: - case WT_TXN_OP_INMEM_ROW: - break; - case WT_TXN_OP_NONE: - case WT_TXN_OP_REF_DELETE: - case WT_TXN_OP_TRUNCATE_COL: - case WT_TXN_OP_TRUNCATE_ROW: - continue; - } - - /* Search for prepared updates. */ - if (F_ISSET(txn, WT_TXN_PREPARE)) - WT_ERR(__txn_search_prepared_op(session, op, &cursor, &upd)); - else - upd = op->u.op_upd; - -#ifdef HAVE_DIAGNOSTIC - op_ts = upd->start_ts; -#endif - /* - * Skip over any aborted update structures, internally created update structures or ones - * from our own transaction. - */ - while (upd != NULL && - (upd->txnid == WT_TXN_ABORTED || upd->txnid == WT_TXN_NONE || upd->txnid == txn->id)) - upd = upd->next; - - /* - * If we didn't track timestamps during update creation, and there are no more updates on - * the chain we won't check any further here. It's not worth reading updates from the disk - * to do this diagnostic checking. - */ - if (upd == NULL) - continue; - - /* - * Check the timestamp on this update with the first valid update in the chain. They're in - * most recent order. - */ - prev_op_ts = upd->start_ts; - prev_op_durable_ts = upd->durable_ts; - - /* - * Check for consistent per-key timestamp usage. If timestamps are or are not used - * originally then they should be used the same way always. For this transaction, timestamps - * are in use anytime the commit timestamp is set. Check timestamps are used in order. - * - * We may see an update restored from the data store or the history store with 0 timestamp - * if that update is behind the oldest timestamp when the page is reconciled. If the update - * is restored from the history store, it is either appended by the prepared rollback or - * rollback to stable. If the update is restored from the data store, it is either - * instantiated along with the prepared stop when the page is read into memory or appended - * by a failed eviction which attempted to write a prepared update to the data store. - */ - op_zero_ts = !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT); - upd_zero_ts = prev_op_durable_ts == WT_TS_NONE; - if (op_zero_ts != upd_zero_ts && - !F_ISSET(upd, WT_UPDATE_RESTORED_FROM_HS | WT_UPDATE_RESTORED_FROM_DS)) { - WT_ERR(__wt_verbose_dump_update(session, upd)); - WT_ERR(__wt_verbose_dump_txn_one(session, session, EINVAL, - "per-key timestamps used inconsistently, dumping relevant information")); - } - /* - * If we aren't using timestamps for this transaction then we are done checking. Don't check - * the timestamp because the one in the transaction is not cleared. - */ - if (op_zero_ts) - continue; - -#ifdef HAVE_DIAGNOSTIC - /* - * Only if the update structure doesn't have a timestamp then use the one in the transaction - * structure. - */ - if (op_ts == WT_TS_NONE) - op_ts = txn->commit_timestamp; -#endif - /* - * Check based on the durable timestamp, but first ensure that it's a stronger check than - * comparing commit timestamps would be. - */ - WT_ASSERT(session, txn->durable_timestamp >= op_ts && prev_op_durable_ts >= prev_op_ts); - if (txn->durable_timestamp < prev_op_durable_ts) - WT_ERR_MSG(session, EINVAL, "out of order commit timestamps"); - } - -#ifndef HAVE_DIAGNOSTIC - WT_UNUSED(prev_op_ts); -#endif - -err: - if (cursor != NULL) - WT_TRET(cursor->close(cursor)); - return (ret); -} -#endif - -/* - * __txn_commit_timestamps_assert -- - * Validate that timestamps provided to commit are legal. - */ -static inline int -__txn_commit_timestamps_assert(WT_SESSION_IMPL *session) -{ - WT_TXN *txn; - bool used_ts; - - txn = session->txn; - used_ts = F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) || F_ISSET(txn, WT_TXN_HAS_TS_DURABLE); - - /* - * Debugging checks on timestamps, if user requested them. We additionally don't expect recovery - * to be using timestamps when applying commits. If recovery is running, skip this assert to - * avoid failing the recovery process. - */ - if (F_ISSET(txn, WT_TXN_TS_WRITE_ALWAYS) && !used_ts && txn->mod_count != 0 && - !F_ISSET(S2C(session), WT_CONN_RECOVERING)) - WT_RET_MSG(session, EINVAL, "commit_timestamp required and none set on this transaction"); - if (F_ISSET(txn, WT_TXN_TS_WRITE_NEVER) && used_ts && txn->mod_count != 0) - WT_RET_MSG( - session, EINVAL, "no commit_timestamp expected and timestamp set on this transaction"); - - if (txn->commit_timestamp > txn->durable_timestamp) - WT_RET_MSG( - session, EINVAL, "transaction with commit timestamp greater than durable timestamp"); - -#ifdef WT_STANDALONE_BUILD - /* If we're not doing any key consistency checking, we're done. */ - if (F_ISSET(txn, WT_TXN_TS_WRITE_ORDERED)) - WT_RET(__txn_commit_timestamps_assert_standalone(session, txn)); -#endif - return (0); -} - /* * __txn_mod_compare -- * Qsort comparison routine for transaction modify list. @@ -1618,13 +1438,12 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_OP *op; WT_UPDATE *upd; wt_timestamp_t candidate_durable_timestamp, prev_durable_timestamp; - uint32_t fileid; - uint8_t previous_state; - u_int i, ft_resolution; #ifdef HAVE_DIAGNOSTIC - u_int prepare_count; + uint32_t prepare_count; #endif - bool locked, prepare, readonly, update_durable_ts; + uint8_t previous_state; + u_int i; + bool cannot_fail, locked, prepare, readonly, update_durable_ts; conn = S2C(session); cursor = NULL; @@ -1633,9 +1452,9 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) #ifdef HAVE_DIAGNOSTIC prepare_count = 0; #endif - locked = false; prepare = F_ISSET(txn, WT_TXN_PREPARE); readonly = txn->mod_count == 0; + cannot_fail = locked = false; /* Permit the commit if the transaction failed, but was read-only. */ WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); @@ -1671,8 +1490,17 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) "durable_timestamp should not be specified for non-prepared transaction"); } - WT_ASSERT(session, - !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) || txn->commit_timestamp <= txn->durable_timestamp); + /* + * Release our snapshot in case it is keeping data pinned (this is particularly important for + * checkpoints). Before releasing our snapshot, copy values into any positioned cursors so they + * don't point to updates that could be freed once we don't have a snapshot. If this transaction + * is prepared, then copying values would have been done during prepare. + */ + if (session->ncursors > 0 && !prepare) { + WT_DIAGNOSTIC_YIELD; + WT_ERR(__wt_session_copy_values(session)); + } + __wt_txn_release_snapshot(session); /* * Resolving prepared updates is expensive. Sort prepared modifications so all updates for each @@ -1681,63 +1509,45 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) if (prepare) __wt_qsort(txn->mod, txn->mod_count, sizeof(WT_TXN_OP), __txn_mod_compare); - WT_ERR(__txn_commit_timestamps_assert(session)); - - /* - * The default sync setting is inherited from the connection, but can be overridden by an - * explicit "sync" setting for this transaction. - */ - WT_ERR(__wt_config_gets_def(session, cfg, "sync", 0, &cval)); - - /* - * If the user chose the default setting, check whether sync is enabled for this transaction - * (either inherited or via begin_transaction). If sync is disabled, clear the field to avoid - * the log write being flushed. - * - * Otherwise check for specific settings. We don't need to check for "on" because that is the - * default inherited from the connection. If the user set anything in begin_transaction, we only - * override with an explicit setting. - */ - if (cval.len == 0) { - if (!FLD_ISSET(txn->txn_logsync, WT_LOG_SYNC_ENABLED) && !F_ISSET(txn, WT_TXN_SYNC_SET)) - txn->txn_logsync = 0; - } else { - /* - * If the caller already set sync on begin_transaction then they should not be using sync on - * commit_transaction. Flag that as an error. - */ - if (F_ISSET(txn, WT_TXN_SYNC_SET)) - WT_ERR_MSG(session, EINVAL, "Sync already set during begin_transaction"); - if (WT_STRING_MATCH("off", cval.str, cval.len)) - txn->txn_logsync = 0; - /* - * We don't need to check for "on" here because that is the default to inherit from the - * connection setting. - */ - } - - /* - * We are about to release the snapshot: copy values into any positioned cursors so they don't - * point to updates that could be freed once we don't have a snapshot. If this transaction is - * prepared, then copying values would have been done during prepare. - */ - if (session->ncursors > 0 && !prepare) { - WT_DIAGNOSTIC_YIELD; - WT_ERR(__wt_session_copy_values(session)); - } - /* If we are logging, write a commit log record. */ if (txn->logrec != NULL) { /* Assert environment and tree are logging compatible, the fast-check is short-hand. */ WT_ASSERT(session, - FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && - !F_ISSET(session, WT_SESSION_NO_LOGGING)); + !F_ISSET(conn, WT_CONN_RECOVERING) && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)); /* - * We are about to block on I/O writing the log. Release our snapshot in case it is keeping - * data pinned. This is particularly important for checkpoints. + * The default sync setting is inherited from the connection, but can be overridden by an + * explicit "sync" setting for this transaction. */ - __wt_txn_release_snapshot(session); + WT_ERR(__wt_config_gets_def(session, cfg, "sync", 0, &cval)); + + /* + * If the user chose the default setting, check whether sync is enabled for this transaction + * (either inherited or via begin_transaction). If sync is disabled, clear the field to + * avoid the log write being flushed. + * + * Otherwise check for specific settings. We don't need to check for "on" because that is + * the default inherited from the connection. If the user set anything in begin_transaction, + * we only override with an explicit setting. + */ + if (cval.len == 0) { + if (!FLD_ISSET(txn->txn_logsync, WT_LOG_SYNC_ENABLED) && !F_ISSET(txn, WT_TXN_SYNC_SET)) + txn->txn_logsync = 0; + } else { + /* + * If the caller already set sync on begin_transaction then they should not be using + * sync on commit_transaction. Flag that as an error. + */ + if (F_ISSET(txn, WT_TXN_SYNC_SET)) + WT_ERR_MSG(session, EINVAL, "sync already set during begin_transaction"); + if (WT_STRING_MATCH("off", cval.str, cval.len)) + txn->txn_logsync = 0; + /* + * We don't need to check for "on" here because that is the default to inherit from the + * connection setting. + */ + } + /* * We hold the visibility lock for reading from the time we write our log record until the * time we release our transaction so that the LSN any checkpoint gets will always reflect @@ -1748,12 +1558,8 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_txn_log_commit(session, cfg)); } - /* Note: we're going to commit: nothing can fail after this point. */ - - /* Process and free updates. */ - ft_resolution = 0; + /* Process updates. */ for (i = 0, op = txn->mod; i < txn->mod_count; i++, op++) { - fileid = op->btree->id; switch (op->type) { case WT_TXN_OP_NONE: break; @@ -1761,9 +1567,9 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) case WT_TXN_OP_BASIC_ROW: case WT_TXN_OP_INMEM_COL: case WT_TXN_OP_INMEM_ROW: - upd = op->u.op_upd; - if (!prepare) { + upd = op->u.op_upd; + /* * Switch reserved operations to abort to simplify obsolete update list truncation. */ @@ -1777,11 +1583,11 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) * transaction timestamp. Those records should already have the original time window * when they are inserted into the history store. */ - if (conn->cache->hs_fileid != 0 && fileid == conn->cache->hs_fileid) + if (conn->cache->hs_fileid != 0 && op->btree->id == conn->cache->hs_fileid) break; __wt_txn_op_set_timestamp(session, op); - WT_ERR(__txn_commit_timestamps_usage_check(session, op, upd)); + __txn_timestamp_usage_check(session, op, upd); } else { /* * If an operation has the key repeated flag set, skip resolving prepared updates as @@ -1795,19 +1601,14 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) } break; case WT_TXN_OP_REF_DELETE: - /* - * Fast-truncate operations are resolved in a second pass after failure is no longer - * possible. - */ - ++ft_resolution; - continue; + __wt_txn_op_set_timestamp(session, op); + break; case WT_TXN_OP_TRUNCATE_COL: case WT_TXN_OP_TRUNCATE_ROW: /* Other operations don't need timestamps. */ break; } - __wt_txn_op_free(session, op); /* If we used the cursor to resolve prepared updates, the key now has been freed. */ if (cursor != NULL) WT_CLEAR(cursor->key); @@ -1818,35 +1619,38 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) cursor = NULL; } +#ifdef HAVE_DIAGNOSTIC + WT_ASSERT(session, txn->prepare_count == prepare_count); + txn->prepare_count = 0; +#endif + /* + * Note: we're going to commit: nothing can fail after this point. Set a check, it's too easy to + * call an error handling macro between here and the end of the function. + */ + cannot_fail = true; + + /* + * Free updates. + * * Resolve any fast-truncate transactions and allow eviction to proceed on instantiated pages. * This isn't done as part of the initial processing because until now the commit could still * switch to an abort. The action allowing eviction to proceed is clearing the WT_UPDATE list, * (if any), associated with the commit. We're the only consumer of that list and we no longer * need it, and eviction knows it means abort or commit has completed on instantiated pages. */ - for (i = 0, op = txn->mod; ft_resolution > 0 && i < txn->mod_count; i++, op++) + for (i = 0, op = txn->mod; i < txn->mod_count; i++, op++) { if (op->type == WT_TXN_OP_REF_DELETE) { - __wt_txn_op_set_timestamp(session, op); - WT_REF_LOCK(session, op->u.ref, &previous_state); if (previous_state == WT_REF_DELETED) op->u.ref->ft_info.del->committed = 1; else __wt_free(session, op->u.ref->ft_info.update); WT_REF_UNLOCK(op->u.ref, previous_state); - - __wt_txn_op_free(session, op); - - --ft_resolution; } - WT_ASSERT(session, ft_resolution == 0); - + __wt_txn_op_free(session, op); + } txn->mod_count = 0; -#ifdef HAVE_DIAGNOSTIC - WT_ASSERT(session, txn->prepare_count == prepare_count); - txn->prepare_count = 0; -#endif /* * If durable is set, we'll try to update the global durable timestamp with that value. If @@ -1892,13 +1696,6 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) } /* - * We're between transactions, if we need to block for eviction, it's a good time to do so. Note - * that we must ignore any error return because the user's data is committed. - */ - if (!readonly) - WT_IGNORE_RET(__wt_cache_eviction_check(session, false, false, NULL)); - - /* * Stable timestamp cannot be concurrently increased greater than or equal to the prepared * transaction's durable timestamp. Otherwise, checkpoint may only write partial updates of the * transaction. @@ -1906,11 +1703,17 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) if (prepare && txn->durable_timestamp <= txn_global->stable_timestamp) { WT_ERR(__wt_verbose_dump_sessions(session, true)); WT_ERR_PANIC(session, WT_PANIC, - "Stable timestamp is increased greater than or equal to the committing prepared " - "transaction's " + "stable timestamp is larger than or equal to the committing prepared transaction's " "durable timestamp"); } + /* + * We're between transactions, if we need to block for eviction, it's a good time to do so. Note + * that we must ignore any error return because the user's data is committed. + */ + if (!readonly) + WT_IGNORE_RET(__wt_cache_eviction_check(session, false, false, NULL)); + return (0); err: @@ -1926,6 +1729,11 @@ err: if (locked) __wt_readunlock(session, &txn_global->visibility_rwlock); + /* Check for a failure after we can no longer fail. */ + if (cannot_fail) + WT_RET_PANIC(session, ret, + "failed to commit a transaction after data corruption point, failing the system"); + /* * Check for a prepared transaction, and quit: we can't ignore the error and we can't roll back * a prepared transaction. @@ -2646,66 +2454,3 @@ __wt_verbose_dump_txn(WT_SESSION_IMPL *session) return (0); } - -/* - * __wt_verbose_dump_update -- - * Output diagnostic information about an update structure. - */ -int -__wt_verbose_dump_update(WT_SESSION_IMPL *session, WT_UPDATE *upd) -{ - char ts_string[2][WT_TS_INT_STRING_SIZE]; - const char *prepare_state, *upd_type; - - if (upd == NULL) { - WT_RET(__wt_msg(session, "NULL update")); - return (0); - } - WT_NOT_READ(upd_type, "WT_UPDATE_INVALID"); - switch (upd->type) { - case WT_UPDATE_INVALID: - upd_type = "WT_UPDATE_INVALID"; - break; - case WT_UPDATE_MODIFY: - upd_type = "WT_UPDATE_MODIFY"; - break; - case WT_UPDATE_RESERVE: - upd_type = "WT_UPDATE_RESERVE"; - break; - case WT_UPDATE_STANDARD: - upd_type = "WT_UPDATE_STANDARD"; - break; - case WT_UPDATE_TOMBSTONE: - upd_type = "WT_UPDATE_TOMBSTONE"; - break; - } - - WT_NOT_READ(prepare_state, "WT_PREPARE_INVALID"); - switch (upd->prepare_state) { - case WT_PREPARE_INIT: - prepare_state = "WT_PREPARE_INIT"; - break; - case WT_PREPARE_INPROGRESS: - prepare_state = "WT_PREPARE_INPROGRESS"; - break; - case WT_PREPARE_LOCKED: - prepare_state = "WT_PREPARE_LOCKED"; - break; - case WT_PREPARE_RESOLVED: - prepare_state = "WT_PREPARE_RESOLVED"; - break; - } - - __wt_errx(session, - "transaction id: %" PRIu64 - ", commit timestamp: %s" - ", durable timestamp: %s" - ", has next: %s" - ", size: %" PRIu32 - ", type: %s" - ", prepare state: %s", - upd->txnid, __wt_timestamp_to_string(upd->start_ts, ts_string[0]), - __wt_timestamp_to_string(upd->durable_ts, ts_string[1]), upd->next == NULL ? "no" : "yes", - upd->size, upd_type, prepare_state); - return (0); -} diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 4c3f1414f91..c6b3f2982b0 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -251,18 +251,11 @@ __wt_txn_log_op(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_ITEM *logrec; WT_TXN *txn; WT_TXN_OP *op; - uint32_t fileid; conn = S2C(session); txn = session->txn; - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) || - F_ISSET(session, WT_SESSION_NO_LOGGING) || - (F_ISSET(S2BT(session), WT_BTREE_NO_LOGGING) && - !FLD_ISSET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE))) - return (0); - /* We'd better have a transaction. */ WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING) && F_ISSET(txn, WT_TXN_HAS_ID)); @@ -391,20 +384,16 @@ int __wt_txn_ts_log(WT_SESSION_IMPL *session) { struct timespec t; - WT_CONNECTION_IMPL *conn; WT_ITEM *logrec; WT_TXN *txn; WT_TXN_SHARED *txn_shared; wt_timestamp_t commit, durable, first_commit, prepare, read; - conn = S2C(session); txn = session->txn; txn_shared = WT_SESSION_TXN_SHARED(session); - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) || - F_ISSET(session, WT_SESSION_NO_LOGGING) || - !FLD_ISSET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE)) - return (0); + /* We'd better have a transaction, but we may not have allocated an ID. */ + WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); /* * There is a rare usage case of a prepared transaction that has no modifications, but then @@ -414,9 +403,6 @@ __wt_txn_ts_log(WT_SESSION_IMPL *session) if (F_ISSET(txn, WT_TXN_PREPARE) && txn->mod_count == 0) return (0); - /* We'd better have a transaction running. */ - WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); - WT_RET(__txn_logrec_init(session)); logrec = txn->logrec; commit = durable = first_commit = prepare = read = WT_TS_NONE; diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 718be8f2a02..4d003706e86 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -802,6 +802,8 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[]) bool rts_executed; conn = S2C(session); + F_SET(conn, WT_CONN_RECOVERING); + WT_CLEAR(r); WT_INIT_LSN(&r.ckpt_lsn); config = NULL; @@ -811,14 +813,12 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[]) was_backup = F_ISSET(conn, WT_CONN_WAS_BACKUP); /* We need a real session for recovery. */ - WT_RET( - __wt_open_internal_session(conn, "txn-recover", false, WT_SESSION_NO_LOGGING, 0, &session)); + WT_RET(__wt_open_internal_session(conn, "txn-recover", false, 0, 0, &session)); r.session = session; WT_MAX_LSN(&r.max_ckpt_lsn); WT_MAX_LSN(&r.max_rec_lsn); conn->txn_global.recovery_timestamp = conn->txn_global.meta_ckpt_timestamp = WT_TS_NONE; - F_SET(conn, WT_CONN_RECOVERING); WT_ERR(__wt_metadata_search(session, WT_METAFILE_URI, &config)); WT_ERR(__recovery_setup_file(&r, WT_METAFILE_URI, config)); WT_ERR(__wt_metadata_cursor_open(session, NULL, &metac)); diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 854bc7ac0ca..07c3c3cb1d3 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -1883,8 +1883,8 @@ __wt_rollback_to_stable(WT_SESSION_IMPL *session, const char *cfg[], bool no_ckp * concurrently. Copy parent session no logging option to the internal session to make sure that * rollback to stable doesn't generate log records. */ - WT_RET(__wt_open_internal_session(S2C(session), "txn rollback_to_stable", true, - F_MASK(session, WT_SESSION_NO_LOGGING), 0, &session)); + WT_RET( + __wt_open_internal_session(S2C(session), "txn rollback_to_stable", true, 0, 0, &session)); WT_STAT_CONN_SET(session, txn_rollback_to_stable_running, 1); WT_WITH_CHECKPOINT_LOCK( diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 0b9b99509f2..99bbc505c7a 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -876,12 +876,14 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CONFIG cparser; WT_CONFIG_ITEM ckey, cval; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; wt_timestamp_t commit_ts, durable_ts, prepare_ts, read_ts; bool set_ts; - set_ts = false; + conn = S2C(session); commit_ts = durable_ts = prepare_ts = read_ts = WT_TS_NONE; + set_ts = false; WT_TRET(__wt_txn_context_check(session, true)); @@ -936,7 +938,9 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) if (prepare_ts != WT_TS_NONE) WT_RET(__wt_txn_set_prepare_timestamp(session, prepare_ts)); - if (set_ts) + /* Timestamps are only logged in debugging mode. */ + if (set_ts && FLD_ISSET(conn->log_flags, WT_CONN_LOG_DEBUG_MODE) && + FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && !F_ISSET(conn, WT_CONN_RECOVERING)) WT_RET(__wt_txn_ts_log(session)); return (0); diff --git a/src/third_party/wiredtiger/test/suite/test_assert01.py b/src/third_party/wiredtiger/test/suite/test_assert01.py deleted file mode 100644 index cb12d8bd030..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_assert01.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_assert01.py -# Timestamps: assert commit settings -# - -from suite_subprocess import suite_subprocess -import wiredtiger, wttest -from wtscenario import make_scenarios - -class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): - base = 'assert01' - base_uri = 'file:' + base - uri_always = base_uri + '.always.wt' - uri_def = base_uri + '.def.wt' - uri_never = base_uri + '.never.wt' - uri_none = base_uri + '.none.wt' - cfg_always = 'verbose=[write_timestamp],write_timestamp_usage=always,assert=(write_timestamp=on)' - cfg_def = '' - cfg_never = 'verbose=(write_timestamp=true),write_timestamp_usage=never,assert=(write_timestamp=on)' - cfg_none = 'assert=(write_timestamp=off)' - - key_format_values = [ - ('column', dict(key_format='r', usestrings=False)), - ('string-row', dict(key_format='S', usestrings=True)) - ] - scenarios = make_scenarios(key_format_values) - - count = 1 - # - # Commit a k/v pair making sure that it detects an error if needed, when - # used with and without a commit timestamp. - # - def insert_check(self, uri, use_ts): - c = self.session.open_cursor(uri) - key = 'key' + str(self.count) if self.usestrings else self.count - val = 'value' + str(self.count) - - # Commit with a timestamp - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(self.count)) - c[key] = val - # All settings other than never should commit successfully - if (use_ts != 'never'): - self.session.commit_transaction() - else: - msg = "/timestamp set on this transaction/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(self.session.commit_transaction(), - 0), msg) - c.close() - self.count += 1 - - # Commit without a timestamp - key = 'key' + str(self.count) if self.usestrings else self.count - val = 'value' + str(self.count) - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key] = val - # All settings other than always should commit successfully - if (use_ts != 'always'): - self.session.commit_transaction() - else: - msg = "/none set on this transaction/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(self.session.commit_transaction(), - 0), msg) - self.count += 1 - c.close() - - def test_commit_timestamp(self): - cfg = 'key_format={},value_format=S,'.format(self.key_format) - - # Create a data item at a timestamp - self.session.create(self.uri_always, cfg + self.cfg_always) - self.session.create(self.uri_def, cfg + self.cfg_def) - self.session.create(self.uri_never, cfg + self.cfg_never) - self.session.create(self.uri_none, cfg + self.cfg_none) - - # Check inserting into each table - self.insert_check(self.uri_always, 'always') - self.insert_check(self.uri_def, 'none') - self.insert_check(self.uri_never, 'never') - self.insert_check(self.uri_none, 'none') - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_assert02.py b/src/third_party/wiredtiger/test/suite/test_assert02.py deleted file mode 100644 index d9bd372f7a9..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_assert02.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_assert02.py -# Timestamps: assert read timestamp settings -# - -from suite_subprocess import suite_subprocess -import wiredtiger, wttest -from wtscenario import make_scenarios - -class test_assert02(wttest.WiredTigerTestCase, suite_subprocess): - - key_format_values = [ - ('column', dict(key_format='r', usestrings=False)), - ('string-row', dict(key_format='S', usestrings=True)) - ] - scenarios = make_scenarios(key_format_values) - - def test_read_timestamp(self): - #if not wiredtiger.diagnostic_build(): - # self.skipTest('requires a diagnostic build') - - base = 'assert02.' - base_uri = 'file:' + base - uri_always = base_uri + '.always.wt' - uri_def = base_uri + '.def.wt' - uri_never = base_uri + '.never.wt' - uri_none = base_uri + '.none.wt' - - cfg = 'key_format={},value_format=S'.format(self.key_format) - cfg_always = cfg + ',write_timestamp_usage=always,assert=(read_timestamp=always)' - cfg_def = cfg - cfg_never = cfg + ',assert=(read_timestamp=never)' - cfg_none = cfg + ',assert=(read_timestamp=none)' - - # Create a data item at a timestamp. - self.session.create(uri_always, cfg_always) - self.session.create(uri_def, cfg_def) - self.session.create(uri_never, cfg_never) - self.session.create(uri_none, cfg_none) - - # Make a key. - key1 = 'key1' if self.usestrings else 1 - - # Insert a data item at timestamp 1. This should work for all. - c_always = self.session.open_cursor(uri_always) - c_def = self.session.open_cursor(uri_def) - c_never = self.session.open_cursor(uri_never) - c_none = self.session.open_cursor(uri_none) - self.session.begin_transaction() - self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) - c_always[key1] = 'value1' - c_def[key1] = 'value1' - c_never[key1] = 'value1' - c_none[key1] = 'value1' - self.session.commit_transaction() - c_always.close() - c_def.close() - c_never.close() - c_none.close() - - # Now that we have a timestamped data, try reading with and without - # the timestamp. - c_always = self.session.open_cursor(uri_always) - c_def = self.session.open_cursor(uri_def) - c_never = self.session.open_cursor(uri_never) - c_none = self.session.open_cursor(uri_none) - - c_always.set_key(key1) - c_def.set_key(key1) - c_never.set_key(key1) - c_none.set_key(key1) - - self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) - c_always.search() - c_def.search() - c_none.search() - self.assertEqual(c_always.get_value(), 'value1') - self.assertEqual(c_def.get_value(), 'value1') - self.assertEqual(c_none.get_value(), 'value1') - - msg = "/timestamp set on this transaction/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(c_never.search(), 0), msg) - self.session.rollback_transaction() - c_always.close() - c_def.close() - c_never.close() - c_none.close() - - # Read in a transaction without a timestamp. - c_always = self.session.open_cursor(uri_always) - c_def = self.session.open_cursor(uri_def) - c_never = self.session.open_cursor(uri_never) - c_none = self.session.open_cursor(uri_none) - - c_always.set_key(key1) - c_def.set_key(key1) - c_never.set_key(key1) - c_none.set_key(key1) - - self.session.begin_transaction() - c_never.search() - c_def.search() - c_none.search() - self.assertEqual(c_never.get_value(), 'value1') - self.assertEqual(c_def.get_value(), 'value1') - self.assertEqual(c_none.get_value(), 'value1') - - msg = "/none set on this transaction/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(c_always.search(), 0), msg) - self.session.rollback_transaction() - c_always.close() - c_def.close() - c_never.close() - c_none.close() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_assert03.py b/src/third_party/wiredtiger/test/suite/test_assert03.py deleted file mode 100644 index fead7358cf5..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_assert03.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_assert03.py -# Test changing assert setting via alter. -# - -from suite_subprocess import suite_subprocess -import wiredtiger, wttest -from wtscenario import make_scenarios - -class test_assert03(wttest.WiredTigerTestCase, suite_subprocess): - base_uri = 'file:assert03.wt' - always = 'write_timestamp_usage=always,assert=(write_timestamp=on)' - never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' - none = 'assert=(write_timestamp=off)' - - key_format_values = [ - ('col-fix', dict(key_format='r', value_format='8t')), - ('col', dict(key_format='r', value_format='S')), - ('row', dict(key_format='S', value_format='S')) - ] - scenarios = make_scenarios(key_format_values) - - def test_assert03(self): - #if not wiredtiger.diagnostic_build(): - # self.skipTest('requires a diagnostic build') - - cfg = 'key_format={},'.format(self.key_format) + 'value_format={}'.format(self.value_format) - key0 = 'key0' if self.key_format == 'S' else 17 - value0 = 'value0' if self.value_format == 'S' else 0x2a - key1 = 'key1' if self.key_format == 'S' else 18 - value1 = 'value1' if self.value_format == 'S' else 0x2b - key2 = 'key2' if self.key_format == 'S' else 19 - value2 = 'value2' if self.value_format == 'S' else 0x2c - key3 = 'key3' if self.key_format == 'S' else 20 - value3 = 'value3' if self.value_format == 'S' else 0x2d - - # Create a data item at the default setting - self.session.create(self.base_uri, cfg) - c = self.session.open_cursor(self.base_uri) - self.session.begin_transaction() - c[key0] = value0 - self.session.commit_transaction() - c.close() - - # Now rotate through the alter settings and verify the data. - # The always setting should fail. - self.session.alter(self.base_uri, self.always) - c = self.session.open_cursor(self.base_uri) - self.session.begin_transaction() - c[key1] = value1 - msg = "/none set on this transaction/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(self.session.commit_transaction(), 0), msg) - c.close() - - # The never and none settings should succeed. - self.session.alter(self.base_uri, self.never) - c = self.session.open_cursor(self.base_uri) - self.session.begin_transaction() - c[key2] = value2 - self.session.commit_transaction() - c.close() - - self.session.alter(self.base_uri, self.none) - c = self.session.open_cursor(self.base_uri) - self.session.begin_transaction() - c[key3] = value3 - self.session.commit_transaction() - c.close() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_assert04.py b/src/third_party/wiredtiger/test/suite/test_assert04.py deleted file mode 100644 index 4f20dd7ff0c..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_assert04.py +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_assert04.py -# Timestamps: verify consistency usage on keys -# - -from suite_subprocess import suite_subprocess -import wiredtiger, wttest -from wtscenario import make_scenarios - -class test_assert04(wttest.WiredTigerTestCase, suite_subprocess): - - key_format_values = [ - ('column', dict(key_format='r', usestrings=False)), - ('string-row', dict(key_format='S', usestrings=True)) - ] - scenarios = make_scenarios(key_format_values) - - def test_timestamp_alter(self): - base = 'assert04' - uri = 'file:' + base - cfg_on = 'write_timestamp_usage=ordered,assert=(write_timestamp=on)' - cfg_off = 'assert=(write_timestamp=off)' - msg_ooo='/out of order/' - msg_usage='/used inconsistently/' - - # Create the table without the key consistency checking turned on. - # Create a few items breaking the rules. Then alter the setting and - # verify the inconsistent usage is detected. - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) - - key_nots = 'key_nots' if self.usestrings else 5 - key_ts1 = 'key_ts1' if self.usestrings else 16 - - # Insert a data item at timestamp 2. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(2)) - c[key_ts1] = 'value2' - self.session.commit_transaction() - c.close() - - # Modify the data item at timestamp 1. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(1)) - c[key_ts1] = 'value1' - self.session.commit_transaction() - c.close() - - # Insert a non-timestamped item. Then modify with a timestamp. And - # again modify without a timestamp. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value_nots' - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(2)) - c[key_nots] = 'value2' - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value_nots2' - self.session.commit_transaction() - c.close() - - # We must move the oldest timestamp forward in order to alter. - # Otherwise alter closing the file will fail with EBUSY. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) - - # Now alter the setting and make sure we detect incorrect usage. - self.session.alter(uri, cfg_on) - - # Detect decreasing timestamp. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(5)) - c[key_ts1] = 'value5' - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(4)) - c[key_ts1] = 'value4' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - - # Detect not using a timestamp. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts1] = 'value_nots3' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - - # Detect using a timestamp on the non-timestamp key. - # We must first use a non timestamped operation on the key - # in order to violate the key consistency condition in the - # following transaction. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value_nots3' - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(3)) - c[key_nots] = 'value3' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts1], 'value5') - self.assertEquals(c[key_nots], 'value_nots3') - c.close() - - # Now alter the setting again and detection is off. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) - self.session.alter(uri, cfg_off) - - # Detection is off we can successfully change the same key with and - # without a timestamp. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value_nots4' - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(6)) - c[key_nots] = 'value6' - self.session.commit_transaction() - c.close() - - def test_timestamp_usage(self): - base = 'assert04' - uri = 'file:' + base - msg_ooo='/out of order/' - msg_usage='/used inconsistently/' - - # Create the table with the key consistency checking turned on. - # That checking will verify any individual key is always or never - # used with a timestamp. And if it is used with a timestamp that - # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format={},value_format=S,write_timestamp_usage=ordered,assert=(write_timestamp=on)'.format(self.key_format)) - - key_nots = 'key_nots' if self.usestrings else 5 - key_ts1 = 'key_ts1' if self.usestrings else 16 - key_ts2 = 'key_ts2' if self.usestrings else 17 - key_ts3 = 'key_ts3' if self.usestrings else 18 - key_ts4 = 'key_ts4' if self.usestrings else 19 - key_ts5 = 'key_ts5' if self.usestrings else 20 - key_ts6 = 'key_ts6' if self.usestrings else 21 - - # Insert a data item at timestamp 2. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(2)) - c[key_ts1] = 'value2' - self.session.commit_transaction() - c.close() - - # Modify the data item at timestamp 1. We should detect it is wrong. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(1)) - c[key_ts1] = 'value1' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - - # Make sure we can successfully add a different key at timestamp 1. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(1)) - c[key_ts2] = 'value1' - self.session.commit_transaction() - c.close() - - # - # Insert key_ts3 at timestamp 10 and key_ts4 at 15. - # Then modify both keys in one transaction at timestamp 13. - # We should not be allowed to modify the one from 15. - # So the whole transaction should fail. - # - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(10)) - c[key_ts3] = 'value10' - self.session.commit_transaction() - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(15)) - c[key_ts4] = 'value15' - self.session.commit_transaction() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(13)) - c[key_ts3] = 'value13' - c[key_ts4] = 'value13' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts3], 'value10') - self.assertEquals(c[key_ts4], 'value15') - c.close() - - # - # Separately, we should be able to update key_ts3 at timestamp 10 - # but not update key_ts4 inserted at timestamp 15. - # - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(13)) - c[key_ts3] = 'value13' - self.session.commit_transaction() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(13)) - c[key_ts4] = 'value13' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - - # Make sure multiple update attempts still fail and eventually - # succeed with a later timestamp. This tests that aborted entries - # in the update chain are not considered for the timestamp check. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(14)) - c[key_ts4] = 'value14' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts4], 'value15') - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(16)) - c[key_ts4] = 'value16' - self.session.commit_transaction() - c.close() - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts4], 'value16') - c.close() - - # Now try to modify a key previously used with timestamps without - # one. We should get the inconsistent usage message. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts4] = 'value_nots' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts4] = 'value_nots' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts4], 'value16') - c.close() - - # Now confirm the other way. Create a key without a timestamp and then - # attempt to modify it with a timestamp. The only error checking that - # makes sense here is the inconsistent usage. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value_nots' - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(16)) - c[key_nots] = 'value16' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value_nots1' - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(17)) - c[key_nots] = 'value17' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_nots], 'value_nots1') - c.close() - - # Confirm it is okay to set the timestamp in the middle or end of the - # transaction. That should set the timestamp for the whole thing. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts5] = 'value_notsyet' - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(20)) - c[key_ts5] = 'value20' - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts5], 'value20') - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts6] = 'value_notsyet' - c[key_ts6] = 'value21_after' - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(21)) - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts6], 'value21_after') - c.close() - - # Confirm it is okay to set the timestamp on the commit call. - # That should set the timestamp for the whole thing. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts6] = 'value_committs1' - c[key_ts6] = 'value22' - self.session.commit_transaction('commit_timestamp=' + - self.timestamp_str(22)) - c.close() - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value23' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction( - 'commit_timestamp=' + self.timestamp_str(23)), msg_usage) - c.close() - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_assert05.py b/src/third_party/wiredtiger/test/suite/test_assert05.py deleted file mode 100644 index 9ca5ee02fff..00000000000 --- a/src/third_party/wiredtiger/test/suite/test_assert05.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-present MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_assert05.py -# Timestamps: assert durable timestamp settings -# - -from suite_subprocess import suite_subprocess -import wttest -from wtscenario import make_scenarios - -class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): - base = 'assert05' - base_uri = 'file:' + base - uri_always = base_uri + '.always.wt' - uri_def = base_uri + '.def.wt' - uri_never = base_uri + '.never.wt' - uri_none = base_uri + '.none.wt' - cfg_always = 'verbose=(write_timestamp=true),write_timestamp_usage=always,assert=(write_timestamp=on)' - cfg_def = '' - cfg_never = 'write_timestamp_usage=never,assert=(write_timestamp=on)' - cfg_none = 'assert=(write_timestamp=off)' - - key_format_values = [ - ('column', dict(key_format='r', usestrings=False)), - ('string-row', dict(key_format='S', usestrings=True)) - ] - scenarios = make_scenarios(key_format_values) - - count = 1 - # - # Commit a k/v pair making sure that it detects an error if needed, when - # used with and without a durable timestamp. - # - def insert_check(self, uri, use_ts): - c = self.session.open_cursor(uri) - key = 'key' + str(self.count) if self.usestrings else self.count - val = 'value' + str(self.count) - - # Commit with a timestamp - self.session.begin_transaction() - c[key] = val - self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(self.count)) - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(self.count)) - self.session.timestamp_transaction( - 'durable_timestamp=' + self.timestamp_str(self.count)) - # All settings other than never should commit successfully - if (use_ts != 'never'): - self.session.commit_transaction() - else: - ''' - Commented out for now: the system panics if we fail after preparing a transaction. - - msg = "/timestamp set on this transaction/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(self.session.commit_transaction(), - 0), msg) - ''' - self.session.rollback_transaction() - c.close() - self.count += 1 - - # Commit without a timestamp - key = 'key' + str(self.count) if self.usestrings else self.count - val = 'value' + str(self.count) - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key] = val - if (use_ts == 'always'): - self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(self.count)) - - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(self.count)) - # All settings other than always should commit successfully - if (use_ts != 'always' and use_ts != 'never'): - self.session.commit_transaction() - else: - ''' - Commented out for now: the system panics if we fail after preparing a transaction. - - msg = "/durable_timestamp is required for a prepared/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda:self.assertEquals(self.session.commit_transaction(), - 0), msg) - ''' - self.session.rollback_transaction() - self.count += 1 - c.close() - - def test_durable_timestamp(self): - cfg = 'key_format={},value_format=S,'.format(self.key_format) - - # Create a data item at a timestamp - self.session.create(self.uri_always, cfg + self.cfg_always) - self.session.create(self.uri_def, cfg + self.cfg_def) - self.session.create(self.uri_never, cfg + self.cfg_never) - self.session.create(self.uri_none, cfg + self.cfg_none) - - # Check inserting into each table - self.insert_check(self.uri_always, 'always') - self.insert_check(self.uri_def, 'none') - self.insert_check(self.uri_never, 'never') - self.insert_check(self.uri_none, 'none') - -if __name__ == '__main__': - wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 942bd4bc84a..f78bffc63ca 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -33,15 +33,18 @@ from suite_subprocess import suite_subprocess import wiredtiger, wttest from wtscenario import make_scenarios +from wtdataset import SimpleDataSet class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): - key_format_values = [ - ('column', dict(key_format='r', usestrings=False)), - ('string-row', dict(key_format='S', usestrings=True)) + ('fix', dict(key_format='r', value_format='8t')), + ('row', dict(key_format='S', value_format='S')), + ('var', dict(key_format='r', value_format='S')), ] scenarios = make_scenarios(key_format_values) + msg_usage='use timestamps once they are first used' + def apply_timestamps(self, timestamp): self.session.prepare_transaction( 'prepare_timestamp=' + self.timestamp_str(timestamp)) @@ -51,400 +54,201 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): 'durable_timestamp=' + self.timestamp_str(timestamp)) def test_timestamp_alter(self): - base = 'assert06' - uri = 'file:' + base + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + cfg_on = 'write_timestamp_usage=ordered,assert=(write_timestamp=on)' cfg_off = 'write_timestamp_usage=never,assert=(write_timestamp=off)' - msg_ooo='/out of order/' - msg_usage='/used inconsistently/' - - key_nots = 'key_nots' if self.usestrings else 5 - key_ts1 = 'key_ts1' if self.usestrings else 16 # Create the table without the key consistency checking turned on. - # Create a few items breaking the rules. Then alter the setting and - # verify the inconsistent usage is detected. - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) - # Insert a data item at timestamp 2. + # Create a few items breaking the rules. + # Then alter the setting and verify the inconsistent usage is detected. + uri = 'file:assert06' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format)) c = self.session.open_cursor(uri) + + # Insert a data item at timestamp 2. + key = ds.key(1) self.session.begin_transaction() - c[key_ts1] = 'value2' + c[key] = ds.value(1) self.apply_timestamps(2) self.session.commit_transaction() - c.close() - # Modify the data item at timestamp 1. - c = self.session.open_cursor(uri) + # Modify the data item at timestamp 1, illegally moving the timestamp backward. self.session.begin_transaction() - c[key_ts1] = 'value1' + c[key] = ds.value(2) self.apply_timestamps(1) self.session.commit_transaction() - c.close() - # Insert a non-timestamped item. Then modify with a timestamp. And - # again modify without a timestamp. - c = self.session.open_cursor(uri) + # Insert a non-timestamped item. + # Then illegally modify with a timestamp. + # Then illegally modify without a timestamp. + key = ds.key(2) self.session.begin_transaction() - c[key_nots] = 'value_nots' + c[key] = ds.value(3) self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_nots] = 'value2' + c[key] = ds.value(4) self.apply_timestamps(2) self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_nots] = 'value_nots2' + c[key] = ds.value(5) self.session.commit_transaction() - c.close() - - # We must move the oldest timestamp forward in order to alter. - # Otherwise alter closing the file will fail with EBUSY. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) # Now alter the setting and make sure we detect incorrect usage. + # We must move the oldest timestamp forward in order to alter, otherwise alter closing the + # file will fail with EBUSY. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) + c.close() self.session.alter(uri, cfg_on) - - # Detect decreasing timestamp. c = self.session.open_cursor(uri) + + # Update at timestamp 5, then detect not using a timestamp. + key = ds.key(3) self.session.begin_transaction() - c[key_ts1] = 'value5' + c[key] = ds.value(6) self.apply_timestamps(5) self.session.commit_transaction() - c.close() - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. - - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_ts1] = 'value4' - self.apply_timestamps(4) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - ''' + c[key] = ds.value(6) + with self.expectedStderrPattern(self.msg_usage): + self.session.commit_transaction() - # Detect not using a timestamp. - c = self.session.open_cursor(uri) + # Detect using a timestamp on a non-timestamp key. We must first use a non-timestamped + # operation on the key in order to violate the key consistency condition in the following + # transaction. + key = ds.key(4) self.session.begin_transaction() - c[key_ts1] = 'value_nots3' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - - # Detect using a timestamp on the non-timestamp key. - # We must first use a non timestamped operation on the key - # in order to violate the key consistency condition in the - # following transaction. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value_nots3' + c[key] = ds.value(7) self.session.commit_transaction() - c.close() - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. - - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_nots] = 'value3' - self.apply_timestamps(3) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - self.session.checkpoint() - ''' + c[key] = ds.value(8) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts1], 'value5') - self.assertEquals(c[key_nots], 'value_nots3') + # Test to make sure that key consistency can be turned off after turning it on. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(4)) c.close() - - # Test to make sure that key consistency can be turned off - # after turning it on. - self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) self.session.alter(uri, cfg_off) - - # Detection is off we can successfully change the same key with and - # without a timestamp. c = self.session.open_cursor(uri) + + # Detection is off we can successfully change the same key with and without a timestamp. + key = ds.key(5) self.session.begin_transaction() - c[key_nots] = 'value_nots4' + c[key] = ds.value(9) self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_nots] = 'value6' + c[key] = ds.value(1) self.apply_timestamps(6) self.session.commit_transaction() - c.close() def test_timestamp_usage(self): - base = 'assert06' - uri = 'file:' + base - msg_ooo='/out of order/' - msg_usage='/used inconsistently/' + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') - key_nots = 'key_nots' if self.usestrings else 5 - key_ts1 = 'key_ts1' if self.usestrings else 16 - key_ts2 = 'key_ts2' if self.usestrings else 17 - key_ts3 = 'key_ts3' if self.usestrings else 18 - key_ts4 = 'key_ts4' if self.usestrings else 19 - key_ts5 = 'key_ts5' if self.usestrings else 20 - key_ts6 = 'key_ts6' if self.usestrings else 21 + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) - # Create the table with the key consistency checking turned on. - # That checking will verify any individual key is always or never - # used with a timestamp. And if it is used with a timestamp that - # the timestamps are in increasing order for that key. - self.session.create(uri, 'key_format={},value_format=S,verbose=(write_timestamp),write_timestamp_usage=ordered,assert=(write_timestamp=on)'.format(self.key_format)) + # Create the table with the key consistency checking turned on. That checking will verify + # any individual key is always or never used with a timestamp. And if it is used with a + # timestamp that the timestamps are in increasing order for that key. + uri = 'file:assert06' + self.session.create(uri, + 'key_format={},value_format={},'.format(self.key_format, self.value_format) + + 'write_timestamp_usage=ordered,assert=(write_timestamp=on)') + c = self.session.open_cursor(uri) # Insert a data item at timestamp 2. - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_ts1] = 'value2' + c[ds.key(1)] = ds.value(1) self.apply_timestamps(2) self.session.commit_transaction() - c.close() - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. - - # Modify the data item at timestamp 1. We should detect it is wrong. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts1] = 'value1' - self.apply_timestamps(1) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - ''' # Make sure we can successfully add a different key at timestamp 1. - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_ts2] = 'value1' + c[ds.key(2)] = ds.value(2) self.apply_timestamps(1) self.session.commit_transaction() - c.close() - # - # Insert key_ts3 at timestamp 10 and key_ts4 at 15. - # Then modify both keys in one transaction at timestamp 13. - # We should not be allowed to modify the one from 15. - # So the whole transaction should fail. - # + # Insert key_ts3 at timestamp 10 and key_ts4 at 15, then modify both keys in one transaction + # at timestamp 13, which should result in an error message. c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_ts3] = 'value10' + c[ds.key(3)] = ds.value(3) self.apply_timestamps(10) self.session.commit_transaction() self.session.begin_transaction() - c[key_ts4] = 'value15' + c[ds.key(4)] = ds.value(4) self.apply_timestamps(15) self.session.commit_transaction() - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts3] = 'value13' - c[key_ts4] = 'value13' - self.apply_timestamps(13) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - ''' - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts3], 'value10') - self.assertEquals(c[key_ts4], 'value15') - c.close() - - # - # Separately, we should be able to update key_ts3 at timestamp 10 - # but not update key_ts4 inserted at timestamp 15. - # - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts3] = 'value13' - self.apply_timestamps(13) - self.session.commit_transaction() - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. - - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_ts4] = 'value13' + c[ds.key(3)] = ds.value(5) + c[ds.key(4)] = ds.value(6) self.apply_timestamps(13) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - ''' - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. + with self.expectedStderrPattern('unexpected timestamp usage'): + self.session.commit_transaction() + self.assertEquals(c[ds.key(3)], ds.value(5)) + self.assertEquals(c[ds.key(4)], ds.value(6)) - # Make sure multiple update attempts still fail and eventually - # succeed with a later timestamp. This tests that aborted entries - # in the update chain are not considered for the timestamp check. - c = self.session.open_cursor(uri) + # Modify a key previously used with timestamps without one. We should get the inconsistent + # usage message. + key = ds.key(5) self.session.begin_transaction() - c[key_ts4] = 'value14' + c[key] = ds.value(7) self.apply_timestamps(14) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_ooo) - c.close() - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts4], 'value15') - c.close() - ''' - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts4] = 'value16' - self.apply_timestamps(16) self.session.commit_transaction() - c.close() - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts4], 'value16') - c.close() - - # Now try to modify a key previously used with timestamps without - # one. We should get the inconsistent usage message. - c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key_ts4] = 'value_nots' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() + c[key] = ds.value(8) + with self.expectedStderrPattern(self.msg_usage): + self.session.commit_transaction() + self.assertEquals(c[key], ds.value(8)) - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts4] = 'value_nots' - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts4], 'value16') - c.close() - - # Now confirm the other way. Create a key without a timestamp and then - # attempt to modify it with a timestamp. The only error checking that - # makes sense here is the inconsistent usage. - c = self.session.open_cursor(uri) + # Set the timestamp in the beginning, middle or end of the transaction. + key = ds.key(6) self.session.begin_transaction() - c[key_nots] = 'value_nots' + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(16)) + c[key] = ds.value(9) self.session.commit_transaction() - c.close() - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. + self.assertEquals(c[key], ds.value(9)) - c = self.session.open_cursor(uri) + key = ds.key(7) self.session.begin_transaction() - c[key_nots] = 'value16' - self.apply_timestamps(16) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - ''' - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value_nots1' + c[key] = ds.value(10) + c[key] = ds.value(11) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(17)) + c[key] = ds.value(12) + c[key] = ds.value(13) self.session.commit_transaction() - c.close() - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. + self.assertEquals(c[key], ds.value(13)) - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value17' - self.apply_timestamps(17) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction(), msg_usage) - c.close() - ''' - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_nots], 'value_nots1') - c.close() - - # Confirm it is okay to set the timestamp in the middle or end of the - # transaction. That should set the timestamp for the whole thing. - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_ts5] = 'value_notsyet' - c[key_ts5] = 'value20' - self.apply_timestamps(20) - self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts5], 'value20') - c.close() - - c = self.session.open_cursor(uri) + key = ds.key(8) self.session.begin_transaction() - c[key_ts6] = 'value_notsyet' - c[key_ts6] = 'value21_after' - self.apply_timestamps(21) + c[key] = ds.value(14) + self.apply_timestamps(18) self.session.commit_transaction() - c.close() - - c = self.session.open_cursor(uri) - self.assertEquals(c[key_ts6], 'value21_after') - c.close() + self.assertEquals(c[key], ds.value(14)) # Confirm it is okay to set the durable timestamp on the commit call. - # That should set the timestamp for the whole thing. - c = self.session.open_cursor(uri) + key = ds.key(9) self.session.begin_transaction() - c[key_ts6] = 'value_committs1' - c[key_ts6] = 'value22' - self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(22)) - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(22)) - self.session.timestamp_transaction( - 'durable_timestamp=' + self.timestamp_str(22)) + c[key] = ds.value(15) + c[key] = ds.value(16) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(22)) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(22)) + self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(22)) self.session.commit_transaction() - c.close() - - ''' - Commented out for now: the system panics if we fail after preparing a transaction. - - c = self.session.open_cursor(uri) - self.session.begin_transaction() - c[key_nots] = 'value23' - self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(23)) - self.session.timestamp_transaction( - 'commit_timestamp=' + self.timestamp_str(23)) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.commit_transaction( - 'durable_timestamp=' + self.timestamp_str(23)), msg_usage) - c.close() - ''' # Confirm that rolling back after preparing doesn't fire an assertion. - c = self.session.open_cursor(uri) + key = ds.key(10) self.session.begin_transaction() - c[key_ts6] = 'value24' - self.session.prepare_transaction( - 'prepare_timestamp=' + self.timestamp_str(24)) + c[key] = ds.value(17) + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(30)) self.session.rollback_transaction() - c.close() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp26.py b/src/third_party/wiredtiger/test/suite/test_timestamp26.py new file mode 100644 index 00000000000..ea7f0ae7c10 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_timestamp26.py @@ -0,0 +1,519 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_timestamp26.py +# Timestamps: assert commit settings +# + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# Test assert always/never settings when associated with write_timestamp_usage. +class test_timestamp26_always_never(wttest.WiredTigerTestCase): + conn_config = 'debug_mode=(corruption_abort=false)' + assert_ts = [ + ('on', dict(assert_ts='on')), + ('off', dict(assert_ts='off')), + ] + commit_ts = [ + ('yes', dict(commit_ts=True)), + ('no', dict(commit_ts=False)), + ] + with_ts = [ + ('yes', dict(with_ts=True)), + ('no', dict(with_ts=False)), + ] + write_timestamp = [ + ('always', dict(write_timestamp='always')), + ('never', dict(write_timestamp='never')), + ] + types = [ + ('fix', dict(key_format='r', value_format='8t')), + ('row', dict(key_format='S', value_format='S')), + ('var', dict(key_format='r', value_format='S')), + ] + scenarios = make_scenarios(types, assert_ts, commit_ts, with_ts, write_timestamp) + + def test_always_never(self): + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + + # Open the object, configuring write_timestamp usage. + uri = 'table:ts' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + ',write_timestamp_usage=' + self.write_timestamp + ',' + + ',assert=(write_timestamp=' + self.assert_ts + ')') + + c = self.session.open_cursor(uri) + self.session.begin_transaction() + c[ds.key(7)] = ds.value(8) + + # Commit with a timestamp. + if self.with_ts: + # Check both an explicit timestamp set and a set at commit. + commit_ts = 'commit_timestamp=' + self.timestamp_str(10) + if not self.commit_ts: + self.session.timestamp_transaction(commit_ts) + commit_ts = '' + + if self.assert_ts == 'off' or self.write_timestamp == 'always': + self.session.commit_transaction(commit_ts) + else: + with self.expectedStderrPattern('set when disallowed'): + self.session.commit_transaction(commit_ts) + + # Commit without a timestamp. + else: + if self.assert_ts == 'off' or self.write_timestamp == 'never': + self.session.commit_transaction() + else: + with self.expectedStderrPattern('timestamp required by table'): + self.session.commit_transaction() + +# Test assert read timestamp settings. +class test_timestamp26_read_timestamp(wttest.WiredTigerTestCase): + read_ts = [ + ('always', dict(read_ts='always')), + ('never', dict(read_ts='never')), + ('none', dict(read_ts='none')), + ] + types = [ + ('fix', dict(key_format='r', value_format='8t')), + ('row', dict(key_format='S', value_format='S')), + ('var', dict(key_format='r', value_format='S')), + ] + scenarios = make_scenarios(types, read_ts) + + def test_read_timestamp(self): + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + + # Open the object, configuring timestamp usage. + uri = 'table:ts' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + ',assert=(read_timestamp=' + self.read_ts + ')') + + c = self.session.open_cursor(uri) + key = ds.key(10) + value = ds.value(10) + + # Insert a data item at a timestamp (although it doesn't really matter). + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(10)) + c[key] = value + self.session.timestamp_transaction() + self.session.commit_transaction() + + # Try reading without a timestamp. + self.session.begin_transaction() + c.set_key(key) + msg = 'read timestamps required and none set' + if self.read_ts != 'always': + self.assertEquals(c.search(), 0) + self.assertEqual(c.get_value(), value) + else: + with self.expectedStderrPattern(msg): + self.assertEquals(c.search(), 0) + self.session.rollback_transaction() + + # Try reading with a timestamp. + self.session.begin_transaction() + self.session.timestamp_transaction('read_timestamp=20') + c.set_key(key) + msg = 'read timestamps disallowed' + if self.read_ts != 'never': + self.assertEquals(c.search(), 0) + self.assertEqual(c.get_value(), value) + else: + with self.expectedStderrPattern(msg): + self.assertEquals(c.search(), 0) + self.session.rollback_transaction() + +# Test alter of timestamp settings. +class test_timestamp26_alter(wttest.WiredTigerTestCase): + start = [ + ('always', dict(init_always=True)), + ('never', dict(init_always=False)), + ] + types = [ + ('fix', dict(key_format='r', value_format='8t')), + ('row', dict(key_format='S', value_format='S')), + ('var', dict(key_format='r', value_format='S')), + ] + scenarios = make_scenarios(types, start) + + # Perform and operation and check the result for failure. + def check(self, ds, uri, willfail): + c = self.session.open_cursor(uri) + self.session.begin_transaction() + c[ds.key(10)] = ds.value(10) + if willfail: + msg = 'timestamp required by table configuration' + with self.expectedStderrPattern(msg): + self.session.commit_transaction() + else: + self.session.commit_transaction() + c.close() + + def test_alter(self): + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + + if self.init_always: + start = 'always' + switch = 'never' + else: + start = 'never' + switch = 'always' + + # Open the object, configuring the initial timestamp usage. + # Check it. + # Switch the object to the opposite usage. + # Check it. + uri = 'table:ts' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + ',' + 'write_timestamp_usage={}'.format(start) + ',assert=(write_timestamp=on)') + self.check(ds, uri, self.init_always) + self.session.alter(uri, 'write_timestamp_usage={}'.format(switch)) + self.check(ds, uri, not self.init_always) + +# Test timestamp settings with inconsistent updates. +class test_timestamp26_inconsistent(wttest.WiredTigerTestCase): + types = [ + ('fix', dict(key_format='r', value_format='8t')), + ('row', dict(key_format='S', value_format='S')), + ('var', dict(key_format='r', value_format='S')), + ] + scenarios = make_scenarios(types) + + def test_ordered(self): + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + + # Create the table without the key consistency checking turned on. + # Create a few items breaking the rules. Then alter the setting and + # verify the inconsistent usage is detected. + uri = 'table:ts' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format)) + + c = self.session.open_cursor(uri) + key = ds.key(10) + + # Insert a data item at timestamp 2. + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(2)) + c[key] = ds.value(10) + self.session.commit_transaction() + + # Update the data item at timestamp 1. + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) + c[key] = ds.value(11) + self.session.commit_transaction() + + key = ds.key(12) + + # Insert a non-timestamped item, then update with a timestamp and then without a timestamp. + self.session.begin_transaction() + c[key] = ds.value(12) + self.session.commit_transaction() + + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(2)) + c[key] = ds.value(13) + self.session.commit_transaction() + + self.session.begin_transaction() + c[key] = ds.value(14) + self.session.commit_transaction() + + # Now alter the setting and make sure we detect incorrect usage. We must move the oldest + # timestamp forward in order to alter, otherwise alter will fail with EBUSY. + c.close() + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) + config = 'assert=(write_timestamp=on)' + self.session.alter(uri, 'write_timestamp_usage=ordered,' + config) + + c = self.session.open_cursor(uri) + key = ds.key(15) + + # Detect decreasing timestamp. + self.session.begin_transaction() + c[key] = ds.value(15) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) + + msg = 'with an older timestamp' + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(14)) + c[key] = ds.value(16) + with self.expectedStderrPattern(msg): + self.session.commit_transaction() + + # Detect not using a timestamp. + msg = 'use timestamps once they are first used' + self.session.begin_transaction() + c[key] = ds.value(17) + with self.expectedStderrPattern(msg): + self.session.commit_transaction() + + # Now alter the setting again and detection is off. + c.close() + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(20)) + self.session.alter(uri, 'assert=(write_timestamp=off)') + c = self.session.open_cursor(uri) + key = ds.key(18) + + # Detection is off we can successfully change the same key with then without a timestamp. + self.session.begin_transaction() + c[key] = ds.value(18) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(21)) + + self.session.begin_transaction() + c[key] = ds.value(19) + self.session.commit_transaction() + c.close() + +# Test timestamp settings with inconsistent updates. +class test_timestamp26_ts_inconsistent(wttest.WiredTigerTestCase): + types = [ + ('fix', dict(key_format='r', value_format='8t')), + ('row', dict(key_format='S', value_format='S')), + ('var', dict(key_format='r', value_format='S')), + ] + scenarios = make_scenarios(types) + + def test_timestamp_inconsistent(self): + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + + # Create the table with the key consistency checking turned on. That checking will verify + # any individual key is always or never used with a timestamp. And if it is used with a + # timestamp that the timestamps are in increasing order for that key. + uri = 'table:ts' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + ',write_timestamp_usage=ordered,assert=(write_timestamp=on)') + + c = self.session.open_cursor(uri) + key = ds.key(1) + + # Insert an item at timestamp 2. + self.session.begin_transaction() + c[key] = ds.value(1) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) + + # Upate the data item at timestamp 1, which should fail. + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) + c[key] = ds.value(2) + with self.expectedStderrPattern('updates a value with an older timestamp'): + self.session.commit_transaction() + + # Make sure we can successfully add a different key at timestamp 1. + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) + c[ds.key(2)] = ds.value(3) + self.session.commit_transaction() + + # Insert key1 at timestamp 10 and key2 at 15. Then update both keys in one transaction at + # timestamp 13, and we should get a complaint about usage. + key1 = ds.key(3) + key2 = ds.key(4) + self.session.begin_transaction() + c[key1] = ds.value(3) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) + self.session.begin_transaction() + c[key2] = ds.value(4) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) + + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(13)) + c[key1] = ds.value(5) + c[key2] = ds.value(6) + with self.expectedStderrPattern('updates a value with an older timestamp'): + self.session.commit_transaction() + self.assertEquals(c[key1], ds.value(5)) + self.assertEquals(c[key2], ds.value(6)) + + # Try to update a key previously used with timestamps without one. We should get the + # inconsistent usage error/message. + def test_timestamp_ts_then_nots(self): + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + + # Create the table with the key consistency checking turned on. That checking will verify + # any individual key is always or never used with a timestamp. And if it is used with a + # timestamp that the timestamps are in increasing order for that key. + uri = 'table:ts' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + ',write_timestamp_usage=ordered,assert=(write_timestamp=on)') + + c = self.session.open_cursor(uri) + key = ds.key(5) + + self.session.begin_transaction() + c[key] = ds.value(11) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) + + self.session.begin_transaction() + c[key] = ds.value(12) + msg_usage ='configured to always use timestamps once they are first used' + with self.expectedStderrPattern(msg_usage): + self.session.commit_transaction() + self.assertEquals(c[key], ds.value(12)) + + # Smoke test setting the timestamp at various points in the transaction. + def test_timestamp_ts_order(self): + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + + # Create the table with the key consistency checking turned on. That checking will verify + # any individual key is always or never used with a timestamp. And if it is used with a + # timestamp that the timestamps are in increasing order for that key. + uri = 'table:ts' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + ',write_timestamp_usage=ordered,assert=(write_timestamp=on)') + + c = self.session.open_cursor(uri) + key1 = ds.key(6) + key2 = ds.key(7) + + self.session.begin_transaction() + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) + c[key1] = ds.value(14) + c[key2] = ds.value(15) + self.session.commit_transaction() + self.assertEquals(c[key1], ds.value(14)) + self.assertEquals(c[key2], ds.value(15)) + + self.session.begin_transaction() + c[key1] = ds.value(16) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(31)) + c[key2] = ds.value(17) + self.session.commit_transaction() + self.assertEquals(c[key1], ds.value(16)) + self.assertEquals(c[key2], ds.value(17)) + + self.session.begin_transaction() + c[key1] = ds.value(18) + c[key2] = ds.value(19) + self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(32)) + self.session.commit_transaction() + self.assertEquals(c[key1], ds.value(18)) + self.assertEquals(c[key2], ds.value(19)) + + self.session.begin_transaction() + c[key1] = ds.value(20) + c[key2] = ds.value(21) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(33)) + self.assertEquals(c[key1], ds.value(20)) + self.assertEquals(c[key2], ds.value(21)) + +# Test that timestamps are ignored in logged files. +class test_timestamp26_log_ts(wttest.WiredTigerTestCase): + # Turn on logging to cause timestamps to be ignored. + conn_config = 'log=(enabled=true)' + + types = [ + ('fix', dict(key_format='r', value_format='8t')), + ('row', dict(key_format='S', value_format='S')), + ('var', dict(key_format='r', value_format='S')), + ] + always = [ + ('always', dict(always=True)), + ('never', dict(always=False)), + ] + scenarios = make_scenarios(types, always) + + # Smoke test that logged files don't complain about timestamps. + def test_log_ts(self): + if wiredtiger.diagnostic_build(): + self.skipTest('requires a non-diagnostic build') + + # Create an object that's never written, it's just used to generate valid k/v pairs. + ds = SimpleDataSet( + self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) + + # Open the object, configuring write_timestamp usage. + uri = 'table:ts' + config = ',write_timestamp_usage=' + config += 'always' if self.always else 'never' + self.session.create(uri, + 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + config + ',assert=(write_timestamp=on)') + + c = self.session.open_cursor(uri) + + # Commit with a timestamp. + self.session.begin_transaction() + c[ds.key(1)] = ds.value(1) + self.session.breakpoint() + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) + + # Commit without a timestamp. + self.session.begin_transaction() + c[ds.key(2)] = ds.value(2) + self.session.commit_transaction() + +if __name__ == '__main__': + wttest.run() |