diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-04-08 16:00:10 +1000 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2019-04-08 16:00:10 +1000 |
commit | fde88e0c1e7ae85801179308a52f261a651bdb7c (patch) | |
tree | ebd10fb5dba849c698b55f3585b91dc90caad97c /src | |
parent | 62c32c9c68ad65157f8fdb46d34bec120a17bbf8 (diff) | |
download | mongo-fde88e0c1e7ae85801179308a52f261a651bdb7c.tar.gz |
Import wiredtiger: 1768d66613fc32b664ac3608a4e740d5e8c6fd0f from branch mongodb-4.2
ref: bb8086c4ed..1768d66613
for: 4.1.10
WT-4639 Use test/format to verify database file portability
WT-4644 column-store missing run-length encoding opportunities
WT-4652 Return an error if durable timestamp is not passed to a prepared transaction.
WT-4687 test/format can try to set oldest timestamp to 0
WT-4693 WT_CONNECTION::reconfigure should not require quiescence when downgraded
Diffstat (limited to 'src')
39 files changed, 950 insertions, 483 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 190044dca0a..ba480ba5cb9 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1341,8 +1341,7 @@ methods = { Config('prepare_timestamp', '', r''' set the prepare timestamp for the updates of the current transaction. The supplied value must not be older than any active read timestamps. - This configuration option is mandatory. See - @ref transaction_timestamps'''), + See @ref transaction_timestamps'''), ]), 'WT_SESSION.timestamp_transaction' : Method([ @@ -1358,10 +1357,14 @@ methods = { current transaction. The value must also not be older than the current stable timestamp. See @ref transaction_timestamps'''), + Config('prepare_timestamp', '', r''' + set the prepare timestamp for the updates of the current transaction. + The supplied value must not be older than any active read timestamps. + See @ref transaction_timestamps'''), Config('read_timestamp', '', r''' read using the specified timestamp. The supplied value must not be older than the current oldest timestamp. This can only be set once - for a transaction. @ref transaction_timestamps'''), + for a transaction. See @ref transaction_timestamps'''), Config('round_to_oldest', 'false', r''' if read timestamp is earlier than oldest timestamp, read timestamp will be rounded to oldest timestamp''', diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index 3bf66a876fd..9f1e1df17f3 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -921,7 +921,7 @@ transaction_ops(WT_SESSION *session_arg) error_check(session->prepare_transaction( session, "prepare_timestamp=2a")); error_check(session->commit_transaction( - session, "commit_timestamp=2b")); + session, "commit_timestamp=2b,durable_timestamp=2b")); /*! [transaction prepare] */ } diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 340e9ea128f..1738efa6d8a 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "bb8086c4ed645854f19554198f391ea8c821f603", + "commit": "1768d66613fc32b664ac3608a4e740d5e8c6fd0f", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 303e4b39a38..09a52300f3b 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -531,6 +531,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_snapshot[] = { static const WT_CONFIG_CHECK confchk_WT_SESSION_timestamp_transaction[] = { { "commit_timestamp", "string", NULL, NULL, NULL, 0 }, { "durable_timestamp", "string", NULL, NULL, NULL, 0 }, + { "prepare_timestamp", "string", NULL, NULL, NULL, 0 }, { "read_timestamp", "string", NULL, NULL, NULL, 0 }, { "round_to_oldest", "boolean", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -1480,9 +1481,9 @@ static const WT_CONFIG_ENTRY config_entries[] = { NULL, 0 }, { "WT_SESSION.timestamp_transaction", - "commit_timestamp=,durable_timestamp=,read_timestamp=," - "round_to_oldest=false", - confchk_WT_SESSION_timestamp_transaction, 4 + "commit_timestamp=,durable_timestamp=,prepare_timestamp=," + "read_timestamp=,round_to_oldest=false", + confchk_WT_SESSION_timestamp_transaction, 5 }, { "WT_SESSION.transaction_sync", "timeout_ms=1200000", diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c index 7df9fa212df..a6783c9fb9b 100644 --- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c +++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c @@ -57,7 +57,7 @@ __wt_conn_compat_config( uint16_t max_major, max_minor, min_major, min_minor; uint16_t rel_major, rel_minor; char *value; - bool txn_active; + bool txn_active, unchg; conn = S2C(session); value = NULL; @@ -65,6 +65,7 @@ __wt_conn_compat_config( max_minor = WT_CONN_COMPAT_NONE; min_major = WT_CONN_COMPAT_NONE; min_minor = WT_CONN_COMPAT_NONE; + unchg = false; WT_RET(__wt_config_gets(session, cfg, "compatibility.release", &cval)); if (cval.len == 0) { @@ -76,21 +77,31 @@ __wt_conn_compat_config( session, &cval, &rel_major, &rel_minor)); /* - * We're doing an upgrade or downgrade, check whether - * transactions are active. + * If the user is running downgraded, then the compatibility + * string is part of the configuration string. Determine if + * the user is actually changing the compatibility. */ - WT_RET(__wt_txn_activity_check(session, &txn_active)); - if (txn_active) - WT_RET_MSG(session, ENOTSUP, - "system must be quiescent" - " for upgrade or downgrade"); + if (reconfig && rel_major == conn->compat_major && + rel_minor == conn->compat_minor) + unchg = true; + else { + /* + * We're doing an upgrade or downgrade, check whether + * transactions are active. + */ + WT_RET(__wt_txn_activity_check(session, &txn_active)); + if (txn_active) + WT_RET_MSG(session, ENOTSUP, + "system must be quiescent" + " for upgrade or downgrade"); + } F_SET(conn, WT_CONN_COMPATIBILITY); } /* - * If we're a reconfigure and the user did not set any compatibility, - * we're done. + * If we're a reconfigure and the user did not set any compatibility + * or did not change the setting, we're done. */ - if (reconfig && !F_ISSET(conn, WT_CONN_COMPATIBILITY)) + if (reconfig && (!F_ISSET(conn, WT_CONN_COMPATIBILITY) || unchg)) goto done; /* diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index 1812bd1a816..0bbe3283dee 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -188,29 +188,15 @@ __wt_timestamp_value_check( */ static inline void __cell_pack_timestamp_value(WT_SESSION_IMPL *session, - uint8_t **pp, wt_timestamp_t *start_tsp, wt_timestamp_t *stop_tsp) + uint8_t **pp, wt_timestamp_t start_ts, wt_timestamp_t stop_ts) { - wt_timestamp_t start_ts, stop_ts; - - __wt_timestamp_value_check(session, *start_tsp, *stop_tsp); + __wt_timestamp_value_check(session, start_ts, stop_ts); /* - * Finalize the timestamps, checking if they're globally visible and - * won't need to be written. - * * TIMESTAMP-FIXME * Values (presumably) have associated transaction IDs, but we haven't - * yet decided how to handle them. For now, ignore them in determining - * value durability. - */ - if (*start_tsp != WT_TS_NONE && - __wt_txn_visible_all(session, WT_TXN_NONE, *start_tsp)) - *start_tsp = WT_TS_NONE; - - start_ts = *start_tsp; - stop_ts = *stop_tsp; - - /* + * yet decided how to handle them. + * * Historic versions and globally visible values don't have associated * timestamps, else set a flag bit and store the packed timestamp pair. */ @@ -302,8 +288,7 @@ __wt_cell_pack_addr(WT_SESSION_IMPL *session, */ static inline size_t __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_CELL *cell, - wt_timestamp_t *start_tsp, wt_timestamp_t *stop_tsp, - uint64_t rle, size_t size) + wt_timestamp_t start_ts, wt_timestamp_t stop_ts, uint64_t rle, size_t size) { uint8_t byte, *p; bool ts; @@ -312,7 +297,7 @@ __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_CELL *cell, p = cell->__chunk; *p = '\0'; - __cell_pack_timestamp_value(session, &p, start_tsp, stop_tsp); + __cell_pack_timestamp_value(session, &p, start_ts, stop_ts); /* * Short data cells without timestamps or run-length encoding have 6 @@ -411,8 +396,7 @@ __wt_cell_pack_value_match(WT_CELL *page_cell, */ static inline size_t __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_CELL *cell, - wt_timestamp_t *start_tsp, wt_timestamp_t *stop_tsp, - uint64_t rle, uint64_t v) + wt_timestamp_t start_ts, wt_timestamp_t stop_ts, uint64_t rle, uint64_t v) { uint8_t *p; @@ -420,7 +404,7 @@ __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_CELL *cell, p = cell->__chunk; *p = '\0'; - __cell_pack_timestamp_value(session, &p, start_tsp, stop_tsp); + __cell_pack_timestamp_value(session, &p, start_ts, stop_ts); if (rle < 2) cell->__chunk[0] |= WT_CELL_VALUE_COPY; /* Type */ @@ -439,7 +423,7 @@ __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_CELL *cell, */ static inline size_t __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, - wt_timestamp_t *start_tsp, wt_timestamp_t *stop_tsp, uint64_t rle) + wt_timestamp_t start_ts, wt_timestamp_t stop_ts, uint64_t rle) { uint8_t *p; @@ -447,7 +431,7 @@ __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, p = cell->__chunk; *p = '\0'; - __cell_pack_timestamp_value(session, &p, start_tsp, stop_tsp); + __cell_pack_timestamp_value(session, &p, start_ts, stop_ts); if (rle < 2) cell->__chunk[0] |= WT_CELL_DEL; /* Type */ @@ -538,8 +522,7 @@ __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size) */ static inline size_t __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type, - wt_timestamp_t *start_tsp, wt_timestamp_t *stop_tsp, - uint64_t rle, size_t size) + wt_timestamp_t start_ts, wt_timestamp_t stop_ts, uint64_t rle, size_t size) { uint8_t *p; @@ -554,7 +537,7 @@ __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type, break; case WT_CELL_VALUE_OVFL: case WT_CELL_VALUE_OVFL_RM: - __cell_pack_timestamp_value(session, &p, start_tsp, stop_tsp); + __cell_pack_timestamp_value(session, &p, start_ts, stop_ts); break; } diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 75275cc6097..1ca81b0b4d9 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -860,12 +860,13 @@ extern int __wt_txn_parse_timestamp(WT_SESSION_IMPL *session, const char *name, extern int __wt_txn_query_timestamp(WT_SESSION_IMPL *session, char *hex_timestamp, const char *cfg[], bool global_txn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_commit_timestamp_validate(WT_SESSION_IMPL *session, const char *name, wt_timestamp_t ts, WT_CONFIG_ITEM *cval, bool durable_ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t commit_ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_set_durable_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t durable_ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_set_prepare_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t prepare_ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t read_ts) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_parse_prepare_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_parse_read_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session); +extern void __wt_txn_publish_commit_timestamp(WT_SESSION_IMPL *session); extern void __wt_txn_clear_commit_timestamp(WT_SESSION_IMPL *session); -extern void __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session); +extern void __wt_txn_publish_read_timestamp(WT_SESSION_IMPL *session); extern void __wt_txn_clear_read_timestamp(WT_SESSION_IMPL *session); extern void __wt_txn_clear_timestamp_queues(WT_SESSION_IMPL *session); diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 166950410d5..7fed51cc76b 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -320,30 +320,31 @@ struct __wt_txn { const char *rollback_reason; /* If rollback, the reason */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_TXN_AUTOCOMMIT 0x000001u -#define WT_TXN_ERROR 0x000002u -#define WT_TXN_HAS_ID 0x000004u -#define WT_TXN_HAS_SNAPSHOT 0x000008u -#define WT_TXN_HAS_TS_COMMIT 0x000010u -#define WT_TXN_HAS_TS_DURABLE 0x000020u -#define WT_TXN_HAS_TS_READ 0x000040u -#define WT_TXN_IGNORE_PREPARE 0x000080u -#define WT_TXN_NAMED_SNAPSHOT 0x000100u -#define WT_TXN_PREPARE 0x000200u -#define WT_TXN_PUBLIC_TS_COMMIT 0x000400u -#define WT_TXN_PUBLIC_TS_READ 0x000800u -#define WT_TXN_READONLY 0x001000u -#define WT_TXN_RUNNING 0x002000u -#define WT_TXN_SYNC_SET 0x004000u -#define WT_TXN_TS_COMMIT_ALWAYS 0x008000u -#define WT_TXN_TS_COMMIT_KEYS 0x010000u -#define WT_TXN_TS_COMMIT_NEVER 0x020000u -#define WT_TXN_TS_DURABLE_ALWAYS 0x040000u -#define WT_TXN_TS_DURABLE_KEYS 0x080000u -#define WT_TXN_TS_DURABLE_NEVER 0x100000u -#define WT_TXN_TS_ROUND_PREPARED 0x200000u -#define WT_TXN_TS_ROUND_READ 0x400000u -#define WT_TXN_UPDATE 0x800000u +#define WT_TXN_AUTOCOMMIT 0x0000001u +#define WT_TXN_ERROR 0x0000002u +#define WT_TXN_HAS_ID 0x0000004u +#define WT_TXN_HAS_SNAPSHOT 0x0000008u +#define WT_TXN_HAS_TS_COMMIT 0x0000010u +#define WT_TXN_HAS_TS_DURABLE 0x0000020u +#define WT_TXN_HAS_TS_PREPARE 0x0000040u +#define WT_TXN_HAS_TS_READ 0x0000080u +#define WT_TXN_IGNORE_PREPARE 0x0000100u +#define WT_TXN_NAMED_SNAPSHOT 0x0000200u +#define WT_TXN_PREPARE 0x0000400u +#define WT_TXN_PUBLIC_TS_COMMIT 0x0000800u +#define WT_TXN_PUBLIC_TS_READ 0x0001000u +#define WT_TXN_READONLY 0x0002000u +#define WT_TXN_RUNNING 0x0004000u +#define WT_TXN_SYNC_SET 0x0008000u +#define WT_TXN_TS_COMMIT_ALWAYS 0x0010000u +#define WT_TXN_TS_COMMIT_KEYS 0x0020000u +#define WT_TXN_TS_COMMIT_NEVER 0x0040000u +#define WT_TXN_TS_DURABLE_ALWAYS 0x0080000u +#define WT_TXN_TS_DURABLE_KEYS 0x0100000u +#define WT_TXN_TS_DURABLE_NEVER 0x0200000u +#define WT_TXN_TS_ROUND_PREPARED 0x0400000u +#define WT_TXN_TS_ROUND_READ 0x0800000u +#define WT_TXN_UPDATE 0x1000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index c7e0352d644..5a091db45a0 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1878,9 +1878,8 @@ struct __wt_session { * @configstart{WT_SESSION.prepare_transaction, see dist/api_data.py} * @config{prepare_timestamp, set the prepare timestamp for the updates * of the current transaction. The supplied value must not be older - * than any active read timestamps. This configuration option is - * mandatory. See @ref transaction_timestamps., a string; default - * empty.} + * than any active read timestamps. See @ref transaction_timestamps., a + * string; default empty.} * @configend * @errors */ @@ -1922,9 +1921,13 @@ struct __wt_session { * timestamp set for the current transaction. The value must also not * be older than the current stable timestamp. See @ref * transaction_timestamps., a string; default empty.} + * @config{prepare_timestamp, set the prepare timestamp for the updates + * of the current transaction. The supplied value must not be older + * than any active read timestamps. See @ref transaction_timestamps., a + * string; default empty.} * @config{read_timestamp, read using the specified timestamp. The * supplied value must not be older than the current oldest timestamp. - * This can only be set once for a transaction. @ref + * This can only be set once for a transaction. See @ref * transaction_timestamps., a string; default empty.} * @config{round_to_oldest, if read timestamp is earlier than oldest * timestamp\, read timestamp will be rounded to oldest timestamp., a diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index 4d03a40b860..8283a75dd18 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -403,9 +403,9 @@ typedef uint64_t wt_timestamp_t; #include "buf.i" /* required by cell.i */ #include "cache.i" /* required by txn.i */ -#include "mutex.i" /* required by txn.i */ -#include "txn.i" /* required by cell.i */ #include "cell.i" /* required by btree.i */ +#include "mutex.i" /* required by btree.i */ +#include "txn.i" /* required by btree.i */ #include "bitstring.i" #include "btree.i" /* required by cursor.i */ diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 4c36a615b0d..f25ada93885 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -299,10 +299,9 @@ static int __rec_cell_build_int_key(WT_SESSION_IMPL *, static int __rec_cell_build_leaf_key(WT_SESSION_IMPL *, WT_RECONCILE *, const void *, size_t, bool *); static int __rec_cell_build_ovfl(WT_SESSION_IMPL *, WT_RECONCILE *, - WT_KV *, uint8_t, wt_timestamp_t *, wt_timestamp_t *, uint64_t); + WT_KV *, uint8_t, wt_timestamp_t, wt_timestamp_t, uint64_t); static int __rec_cell_build_val(WT_SESSION_IMPL *, WT_RECONCILE *, - const void *, size_t, wt_timestamp_t *, wt_timestamp_t *, - uint64_t); + const void *, size_t, wt_timestamp_t, wt_timestamp_t, uint64_t); static void __rec_cleanup(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_col_fix(WT_SESSION_IMPL *, WT_RECONCILE *, WT_REF *); static int __rec_col_fix_slvg(WT_SESSION_IMPL *, @@ -1407,6 +1406,20 @@ __rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, } else upd_select->start_ts = upd_select->stop_ts = upd_select->upd->start_ts; + + /* + * Finalize the timestamps, checking if the update is globally + * visible and timestamps won't need to be written. + * + * TIMESTAMP-FIXME + * Values (presumably) have associated transaction IDs, but we + * haven't yet decided how to handle them. For now, ignore them + * in determining value durability. + */ + if (upd_select->start_ts != WT_TS_NONE && + __wt_txn_visible_all( + session, WT_TXN_NONE, upd_select->start_ts)) + upd_select->start_ts = WT_TS_NONE; } /* @@ -1993,8 +2006,7 @@ __rec_image_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_KV *kv) */ static int __rec_dict_replace(WT_SESSION_IMPL *session, WT_RECONCILE *r, - wt_timestamp_t *start_tsp, wt_timestamp_t *stop_tsp, - uint64_t rle, WT_KV *val) + wt_timestamp_t start_ts, wt_timestamp_t stop_ts, uint64_t rle, WT_KV *val) { WT_DICTIONARY *dp; uint64_t offset; @@ -2037,7 +2049,7 @@ __rec_dict_replace(WT_SESSION_IMPL *session, WT_RECONCILE *r, offset = (uint64_t)WT_PTRDIFF(r->first_free, (uint8_t *)r->cur_ptr->image.mem + dp->offset); val->len = val->cell_len = __wt_cell_pack_copy( - session, &val->cell, start_tsp, stop_tsp, rle, offset); + session, &val->cell, start_ts, stop_ts, rle, offset); val->buf.data = NULL; val->buf.size = 0; } @@ -3541,21 +3553,18 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_CURSOR *cursor; WT_KV *key, *val; WT_RECONCILE *r; - wt_timestamp_t start_ts, stop_ts; bool ovfl_key; r = cbulk->reconcile; btree = S2BT(session); cursor = &cbulk->cbt.iface; - start_ts = WT_TS_NONE; - stop_ts = WT_TS_MAX; key = &r->k; val = &r->v; WT_RET(__rec_cell_build_leaf_key(session, r, /* Build key cell */ cursor->key.data, cursor->key.size, &ovfl_key)); WT_RET(__rec_cell_build_val(session, r, /* Build value cell */ - cursor->value.data, cursor->value.size, &start_ts, &stop_ts, 0)); + cursor->value.data, cursor->value.size, WT_TS_NONE, WT_TS_MAX, 0)); /* Boundary: split or write the page. */ if (WT_CROSSING_SPLIT_BND(r, key->len + val->len)) { @@ -3582,10 +3591,10 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) r->all_empty_value = false; if (btree->dictionary) WT_RET(__rec_dict_replace( - session, r, &start_ts, &stop_ts, 0, val)); + session, r, WT_TS_NONE, WT_TS_MAX, 0, val)); __rec_image_copy(session, r, val); } - __rec_addr_ts_update(r, start_ts, start_ts, stop_ts); + __rec_addr_ts_update(r, WT_TS_NONE, WT_TS_NONE, WT_TS_MAX); /* Update compression state. */ __rec_key_state_update(r, ovfl_key); @@ -3703,17 +3712,14 @@ __wt_bulk_insert_var( WT_BTREE *btree; WT_KV *val; WT_RECONCILE *r; - wt_timestamp_t start_ts, stop_ts; r = cbulk->reconcile; btree = S2BT(session); - start_ts = WT_TS_NONE; - stop_ts = WT_TS_MAX; val = &r->v; if (deleted) { val->cell_len = __wt_cell_pack_del( - session, &val->cell, &start_ts, &stop_ts, cbulk->rle); + session, &val->cell, WT_TS_NONE, WT_TS_MAX, cbulk->rle); val->buf.data = NULL; val->buf.size = 0; val->len = val->cell_len; @@ -3725,7 +3731,7 @@ __wt_bulk_insert_var( */ WT_RET(__rec_cell_build_val(session, r, cbulk->last.data, cbulk->last.size, - &start_ts, &stop_ts, cbulk->rle)); + WT_TS_NONE, WT_TS_MAX, cbulk->rle)); /* Boundary: split or write the page. */ if (WT_CROSSING_SPLIT_BND(r, val->len)) @@ -3734,9 +3740,9 @@ __wt_bulk_insert_var( /* Copy the value onto the page. */ if (btree->dictionary) WT_RET(__rec_dict_replace( - session, r, &start_ts, &stop_ts, cbulk->rle, val)); + session, r, WT_TS_NONE, WT_TS_MAX, cbulk->rle, val)); __rec_image_copy(session, r, val); - __rec_addr_ts_update(r, start_ts, start_ts, stop_ts); + __rec_addr_ts_update(r, WT_TS_NONE, WT_TS_NONE, WT_TS_MAX); /* Update the starting record number in case we split. */ r->recno += cbulk->rle; @@ -4164,19 +4170,19 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, if (deleted) { val->cell_len = __wt_cell_pack_del( - session, &val->cell, &start_ts, &stop_ts, rle); + session, &val->cell, start_ts, stop_ts, rle); val->buf.data = NULL; val->buf.size = 0; val->len = val->cell_len; } else if (overflow_type) { val->cell_len = __wt_cell_pack_ovfl(session, &val->cell, - WT_CELL_VALUE_OVFL, &start_ts, &stop_ts, rle, value->size); + WT_CELL_VALUE_OVFL, start_ts, stop_ts, rle, value->size); val->buf.data = value->data; val->buf.size = value->size; val->len = val->cell_len + value->size; } else WT_RET(__rec_cell_build_val(session, - r, value->data, value->size, &start_ts, &stop_ts, rle)); + r, value->data, value->size, start_ts, stop_ts, rle)); /* Boundary: split or write the page. */ if (__rec_need_split(r, val->len)) @@ -4185,7 +4191,7 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, /* Copy the value onto the page. */ if (!deleted && !overflow_type && btree->dictionary) WT_RET(__rec_dict_replace( - session, r, &start_ts, &stop_ts, rle, val)); + session, r, start_ts, stop_ts, rle, val)); __rec_image_copy(session, r, val); __rec_addr_ts_update(r, start_ts, start_ts, stop_ts); @@ -4298,15 +4304,11 @@ __rec_col_var(WT_SESSION_IMPL *session, WT_COL_FOREACH(page, cip, i) { ovfl_state = OVFL_IGNORE; if ((cell = WT_COL_PTR(page, cip)) == NULL) { - start_ts = WT_TS_NONE; - stop_ts = WT_TS_MAX; nrepeat = 1; ins = NULL; orig_deleted = true; } else { __wt_cell_unpack(session, page, cell, vpack); - start_ts = vpack->start_ts; - stop_ts = vpack->stop_ts; nrepeat = __wt_cell_rle(vpack); ins = WT_SKIP_FIRST(WT_COL_UPDATE(page, cip)); @@ -4360,6 +4362,8 @@ record_loop: /* */ for (n = 0; n < nrepeat; n += repeat_count, src_recno += repeat_count) { + start_ts = vpack->start_ts; + stop_ts = vpack->stop_ts; upd = NULL; if (ins != NULL && WT_INSERT_RECNO(ins) == src_recno) { WT_ERR(__rec_upd_select( @@ -5206,8 +5210,8 @@ __rec_row_leaf(WT_SESSION_IMPL *session, p = tmpval->data; size = tmpval->size; } - WT_ERR(__rec_cell_build_val(session, - r, p, size, &start_ts, &stop_ts, 0)); + WT_ERR(__rec_cell_build_val( + session, r, p, size, start_ts, stop_ts, 0)); dictionary = true; } else if (vpack->raw == WT_CELL_VALUE_OVFL_RM) { /* @@ -5253,7 +5257,7 @@ __rec_row_leaf(WT_SESSION_IMPL *session, */ WT_ERR(__rec_cell_build_val(session, r, "ovfl-unused", strlen("ovfl-unused"), - &start_ts, &stop_ts, 0)); + start_ts, stop_ts, 0)); } else { val->buf.data = vpack->cell; val->buf.size = __wt_cell_total_len(vpack); @@ -5281,14 +5285,14 @@ __rec_row_leaf(WT_SESSION_IMPL *session, WT_ERR(__rec_cell_build_val(session, r, cbt->iface.value.data, cbt->iface.value.size, - &start_ts, &stop_ts, 0)); + start_ts, stop_ts, 0)); dictionary = true; break; case WT_UPDATE_STANDARD: /* Take the value from the update. */ WT_ERR(__rec_cell_build_val(session, r, upd->data, upd->size, - &start_ts, &stop_ts, 0)); + start_ts, stop_ts, 0)); dictionary = true; break; case WT_UPDATE_TOMBSTONE: @@ -5443,7 +5447,7 @@ build: r->all_empty_value = false; if (dictionary && btree->dictionary) WT_ERR(__rec_dict_replace( - session, r, &start_ts, &stop_ts, 0, val)); + session, r, start_ts, stop_ts, 0, val)); __rec_image_copy(session, r, val); } __rec_addr_ts_update(r, start_ts, start_ts, stop_ts); @@ -5530,12 +5534,12 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) session, cbt, upd, F_ISSET(r, WT_REC_VISIBLE_ALL))); WT_RET(__rec_cell_build_val(session, r, cbt->iface.value.data, cbt->iface.value.size, - &start_ts, &stop_ts, 0)); + start_ts, stop_ts, 0)); break; case WT_UPDATE_STANDARD: /* Take the value from the update. */ WT_RET(__rec_cell_build_val(session, r, - upd->data, upd->size, &start_ts, &stop_ts, 0)); + upd->data, upd->size, start_ts, stop_ts, 0)); break; case WT_UPDATE_TOMBSTONE: continue; @@ -5573,7 +5577,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) r->all_empty_value = false; if (btree->dictionary) WT_RET(__rec_dict_replace( - session, r, &start_ts, &stop_ts, 0, val)); + session, r, start_ts, stop_ts, 0, val)); __rec_image_copy(session, r, val); } __rec_addr_ts_update(r, start_ts, start_ts, stop_ts); @@ -5997,13 +6001,11 @@ __rec_cell_build_int_key(WT_SESSION_IMPL *session, { WT_BTREE *btree; WT_KV *key; - wt_timestamp_t start_ts, stop_ts; *is_ovflp = false; btree = S2BT(session); key = &r->k; - start_ts = stop_ts = WT_TS_NONE; /* Keys aren't timestamped. */ /* Copy the bytes into the "current" and key buffers. */ WT_RET(__wt_buf_set(session, r->cur, data, size)); @@ -6015,7 +6017,7 @@ __rec_cell_build_int_key(WT_SESSION_IMPL *session, *is_ovflp = true; return (__rec_cell_build_ovfl(session, r, - key, WT_CELL_KEY_OVFL, &start_ts, &stop_ts, 0)); + key, WT_CELL_KEY_OVFL, WT_TS_NONE, WT_TS_NONE, 0)); } key->cell_len = __wt_cell_pack_int_key(&key->cell, key->buf.size); @@ -6035,7 +6037,6 @@ __rec_cell_build_leaf_key(WT_SESSION_IMPL *session, { WT_BTREE *btree; WT_KV *key; - wt_timestamp_t start_ts, stop_ts; size_t pfx_max; const uint8_t *a, *b; uint8_t pfx; @@ -6043,7 +6044,6 @@ __rec_cell_build_leaf_key(WT_SESSION_IMPL *session, *is_ovflp = false; btree = S2BT(session); - start_ts = stop_ts = WT_TS_NONE; /* Keys aren't timestamped. */ key = &r->k; pfx = 0; @@ -6116,7 +6116,7 @@ __rec_cell_build_leaf_key(WT_SESSION_IMPL *session, *is_ovflp = true; return (__rec_cell_build_ovfl(session, r, key, - WT_CELL_KEY_OVFL, &start_ts, &stop_ts, 0)); + WT_CELL_KEY_OVFL, WT_TS_NONE, WT_TS_NONE, 0)); } return ( __rec_cell_build_leaf_key(session, r, NULL, 0, is_ovflp)); @@ -6194,7 +6194,7 @@ __rec_cell_build_addr(WT_SESSION_IMPL *session, static int __rec_cell_build_val(WT_SESSION_IMPL *session, WT_RECONCILE *r, const void *data, size_t size, - wt_timestamp_t *start_tsp, wt_timestamp_t *stop_tsp, uint64_t rle) + wt_timestamp_t start_ts, wt_timestamp_t stop_ts, uint64_t rle) { WT_BTREE *btree; WT_KV *val; @@ -6223,11 +6223,11 @@ __rec_cell_build_val(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_STAT_DATA_INCR(session, rec_overflow_value); return (__rec_cell_build_ovfl(session, r, - val, WT_CELL_VALUE_OVFL, start_tsp, stop_tsp, rle)); + val, WT_CELL_VALUE_OVFL, start_ts, stop_ts, rle)); } } val->cell_len = __wt_cell_pack_value( - session, &val->cell, start_tsp, stop_tsp, rle, val->buf.size); + session, &val->cell, start_ts, stop_ts, rle, val->buf.size); val->len = val->cell_len + val->buf.size; return (0); @@ -6240,7 +6240,7 @@ __rec_cell_build_val(WT_SESSION_IMPL *session, WT_RECONCILE *r, static int __rec_cell_build_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_KV *kv, uint8_t type, - wt_timestamp_t *start_tsp, wt_timestamp_t *stop_tsp, uint64_t rle) + wt_timestamp_t start_ts, wt_timestamp_t stop_ts, uint64_t rle) { WT_BM *bm; WT_BTREE *btree; @@ -6300,7 +6300,7 @@ __rec_cell_build_ovfl(WT_SESSION_IMPL *session, /* Build the cell and return. */ kv->cell_len = __wt_cell_pack_ovfl( - session, &kv->cell, type, start_tsp, stop_tsp, rle, kv->buf.size); + session, &kv->cell, type, start_ts, stop_ts, rle, kv->buf.size); kv->len = kv->cell_len + kv->buf.size; err: __wt_scr_free(session, &tmp); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 364474dd362..9967dc3b2b3 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -483,6 +483,7 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CONFIG_ITEM cval; WT_TXN *txn; + wt_timestamp_t read_ts; txn = &session->txn; @@ -531,6 +532,11 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) if (cval.val) F_SET(txn, WT_TXN_IGNORE_PREPARE); + /* Check if read timestamp to be rounded up to the oldest timestamp. */ + WT_RET(__wt_config_gets_def(session, cfg, "round_to_oldest", 0, &cval)); + if (cval.val) + F_SET(txn, WT_TXN_TS_ROUND_READ); + /* * Check if the prepare timestamp and the commit timestamp of a * prepared transaction need to be rounded up. @@ -539,18 +545,19 @@ __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) session, cfg, "roundup_timestamps.prepared", 0, &cval)); if (cval.val) F_SET(txn, WT_TXN_TS_ROUND_PREPARED); - else - F_CLR(txn, WT_TXN_TS_ROUND_PREPARED); /* Check if read timestamp needs to be rounded up. */ WT_RET(__wt_config_gets_def( session, cfg, "roundup_timestamps.read", 0, &cval)); if (cval.val) F_SET(txn, WT_TXN_TS_ROUND_READ); - else - F_CLR(txn, WT_TXN_TS_ROUND_READ); - WT_RET(__wt_txn_parse_read_timestamp(session, cfg)); + WT_RET(__wt_config_gets_def(session, cfg, "read_timestamp", 0, &cval)); + if (cval.len != 0) { + WT_RET(__wt_txn_parse_timestamp( + session, "read", &read_ts, &cval)); + WT_RET(__wt_txn_set_read_timestamp(session, read_ts)); + } return (0); } @@ -638,6 +645,8 @@ __wt_txn_release(WT_SESSION_IMPL *session) /* Ensure the transaction flags are cleared on exit */ txn->flags = 0; + txn->prepare_timestamp = WT_TS_NONE; + txn->durable_timestamp = WT_TS_NONE; } /* @@ -792,7 +801,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_GLOBAL *txn_global; WT_TXN_OP *op; WT_UPDATE *upd; - wt_timestamp_t prev_commit_timestamp, ts; + wt_timestamp_t prev_commit_timestamp; uint32_t fileid; u_int i; bool locked, prepare, readonly, update_timestamp; @@ -810,6 +819,7 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) readonly = txn->mod_count == 0; prepare = F_ISSET(txn, WT_TXN_PREPARE); + /* * Clear the prepared round up flag if the transaction is not prepared. * There is no rounding up to do in that case. @@ -817,52 +827,38 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) if (!prepare) F_CLR(txn, WT_TXN_TS_ROUND_PREPARED); - /* Look for a commit timestamp. */ - WT_ERR( - __wt_config_gets_def(session, cfg, "commit_timestamp", 0, &cval)); - if (cval.len != 0) { - WT_ERR(__wt_txn_parse_timestamp(session, "commit", &ts, &cval)); - /* - * For prepared transactions commit timestamp could be earlier - * than stable timestamp. - */ - WT_ERR(__wt_txn_commit_timestamp_validate( - session, "commit", ts, &cval, !prepare)); - txn->commit_timestamp = ts; - __wt_txn_set_commit_timestamp(session); - if (!prepare) - txn->durable_timestamp = txn->commit_timestamp; - } + /* Set the commit and the durable timestamps. */ + WT_ERR( __wt_txn_set_timestamp(session, cfg)); + + if (prepare) { + if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) + WT_ERR_MSG(session, EINVAL, + "commit_timestamp is required for a prepared " + "transaction"); - if (prepare && !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) - WT_ERR_MSG(session, EINVAL, - "commit_timestamp is required for a prepared transaction"); + if (!F_ISSET(txn, WT_TXN_HAS_TS_DURABLE)) + WT_ERR_MSG(session, EINVAL, + "durable_timestamp is required for a prepared " + "transaction"); - /* - * Durable timestamp is required for a prepared transaction. - * If durable timestamp is not given, commit timestamp will be - * considered as durable timestamp. We don't flag error if durable - * timestamp is not specified for prepared transactions, but will flag - * error if durable timestamp is specified for non-prepared - * transactions. - */ - WT_ERR(__wt_config_gets_def( - session, cfg, "durable_timestamp", 0, &cval)); - if (cval.len != 0) { - if (!prepare) + WT_ASSERT(session, + txn->prepare_timestamp <= txn->commit_timestamp); + } else { + if (F_ISSET(txn, WT_TXN_HAS_TS_PREPARE)) WT_ERR_MSG(session, EINVAL, - "durable_timestamp should not be given for " - "non-prepared transaction"); + "prepare timestamp is set for non-prepared " + "transaction"); - WT_ERR(__wt_txn_parse_timestamp( - session, "durable", &ts, &cval)); - /* Durable timestamp should be later than stable timestamp. */ - F_SET(txn, WT_TXN_HAS_TS_DURABLE); - txn->durable_timestamp = ts; - WT_ERR(__wt_txn_commit_timestamp_validate( - session, "durable", ts, &cval, true)); + if (F_ISSET(txn, WT_TXN_HAS_TS_DURABLE)) + WT_ERR_MSG(session, EINVAL, + "durable_timestamp should not be specified for " + "non-prepared transaction"); } + if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) + WT_ASSERT(session, + txn->commit_timestamp <= txn->durable_timestamp); + WT_ERR(__txn_commit_timestamps_assert(session)); /* @@ -1082,10 +1078,11 @@ __wt_txn_prepare(WT_SESSION_IMPL *session, const char *cfg[]) /* Transaction should not have updated any of the logged tables. */ WT_ASSERT(session, txn->logrec == NULL); - WT_RET(__wt_txn_context_check(session, true)); + /* Set the prepare timestamp. */ + WT_RET(__wt_txn_set_timestamp(session, cfg)); - /* Parse and validate the prepare timestamp. */ - WT_RET(__wt_txn_parse_prepare_timestamp(session, cfg)); + if (!F_ISSET(txn, WT_TXN_HAS_TS_PREPARE)) + WT_RET_MSG(session, EINVAL, "prepare timestamp is not set"); /* * We are about to release the snapshot: copy values into any diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index f4e340535ef..7a502265602 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -274,6 +274,13 @@ __txn_global_query_timestamp( break; } __wt_readunlock(session, &txn_global->commit_timestamp_rwlock); + + /* + * If a transaction is committing with timestamp 1, we could + * return zero here, which is unexpected. Fail instead. + */ + if (ts == 0) + return (WT_NOTFOUND); } else if (WT_STRING_MATCH("last_checkpoint", cval.str, cval.len)) /* Read-only value forever. No lock needed. */ ts = txn_global->last_ckpt_timestamp; @@ -592,15 +599,16 @@ set: __wt_writelock(session, &txn_global->rwlock); } /* - * __wt_txn_commit_timestamp_validate -- - * Validate a timestamp to be not older than running transaction commit - * timestamp and running transaction prepare timestamp. Validate a durable - * timestamp to be not older than the global oldest and global stable - * timestamp. + * __wt_txn_set_commit_timestamp -- + * Validate the commit timestamp of a transaction. + * If the commit timestamp is less than the oldest timestamp and + * transaction is configured to roundup timestamps of a prepared + * transaction, then we will roundup the commit timestamp to the prepare + * timestamp of the transaction. */ int -__wt_txn_commit_timestamp_validate(WT_SESSION_IMPL *session, const char *name, - wt_timestamp_t ts, WT_CONFIG_ITEM *cval, bool durable_ts) +__wt_txn_set_commit_timestamp( + WT_SESSION_IMPL *session, wt_timestamp_t commit_ts) { WT_TXN *txn = &session->txn; WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; @@ -608,13 +616,11 @@ __wt_txn_commit_timestamp_validate(WT_SESSION_IMPL *session, const char *name, char ts_string[2][WT_TS_INT_STRING_SIZE]; bool has_oldest_ts, has_stable_ts; - /* - * Added this redundant initialization to circumvent build failure. - */ + /* Added this redundant initialization to circumvent build failure. */ oldest_ts = stable_ts = 0; /* * Compare against the oldest and the stable timestamp. Return an error - * if the given timestamp is older than oldest and/or stable timestamp. + * if the given timestamp is less than oldest and/or stable timestamp. */ has_oldest_ts = txn_global->has_oldest_timestamp; if (has_oldest_ts) @@ -623,166 +629,166 @@ __wt_txn_commit_timestamp_validate(WT_SESSION_IMPL *session, const char *name, if (has_stable_ts) stable_ts = txn_global->stable_timestamp; - if (durable_ts && has_oldest_ts && ts < oldest_ts) { - __wt_timestamp_to_string(oldest_ts, ts_string[0]); - WT_RET_MSG(session, EINVAL, - "%s timestamp %.*s older than oldest timestamp %s", - name, (int)cval->len, cval->str, ts_string[0]); - } - if (durable_ts && has_stable_ts && ts < stable_ts) { - __wt_timestamp_to_string(stable_ts, ts_string[0]); - WT_RET_MSG(session, EINVAL, - "%s timestamp %.*s older than stable timestamp %s", - name, (int)cval->len, cval->str, ts_string[0]); - } - - /* - * Compare against the commit timestamp of the current transaction. - * Return an error if the given timestamp is older than the first - * commit timestamp. - */ - if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && - ts < txn->first_commit_timestamp) { - __wt_timestamp_to_string( - txn->first_commit_timestamp, ts_string[0]); - WT_RET_MSG(session, EINVAL, - "%s timestamp %.*s older than the first " - "commit timestamp %s for this transaction", - name, (int)cval->len, cval->str, ts_string[0]); - } + if (!F_ISSET(txn, WT_TXN_HAS_TS_PREPARE)) { + /* + * For a non-prepared transactions the commit timestamp should + * not be less than the stable timestamp. + */ + if (has_oldest_ts && commit_ts < oldest_ts) { + __wt_timestamp_to_string(commit_ts, ts_string[0]); + __wt_timestamp_to_string( + oldest_ts, ts_string[1]); + WT_RET_MSG(session, EINVAL, + "commit timestamp %s is less than the oldest " + "timestamp %s", + ts_string[0], ts_string[1]); + } - /* - * Compare against the prepare timestamp of the current transaction. - * Return an error if the given timestamp is older than the prepare - * timestamp. - * - * If roundup timestamps is configured, the commit timestamp will be - * rounded up to the prepare timestamp. - */ - if (F_ISSET(txn, WT_TXN_PREPARE) && - !F_ISSET(txn, WT_TXN_TS_ROUND_PREPARED) && - ts < txn->prepare_timestamp) { - __wt_timestamp_to_string(txn->prepare_timestamp, ts_string[0]); - WT_RET_MSG(session, EINVAL, - "%s timestamp %.*s older than the prepare " - "timestamp %s for this transaction", - name, (int)cval->len, cval->str, ts_string[0]); - } + if (has_stable_ts && commit_ts < stable_ts) { + __wt_timestamp_to_string(commit_ts, ts_string[0]); + __wt_timestamp_to_string( + oldest_ts, ts_string[1]); + WT_RET_MSG(session, EINVAL, + "commit timestamp %s is less than the stable " + "timestamp %s", + ts_string[0], ts_string[1]); + } - if (F_ISSET(txn, WT_TXN_HAS_TS_DURABLE) && - txn->durable_timestamp < txn->commit_timestamp) { - __wt_timestamp_to_string(txn->durable_timestamp, ts_string[0]); - __wt_timestamp_to_string(txn->commit_timestamp, ts_string[1]); - WT_RET_MSG(session, EINVAL, - "%s timestamp %s older than the commit timestamp %s " - "for this transaction", - name, ts_string[0], ts_string[1]); + /* + * Compare against the commit timestamp of the current + * transaction. Return an error if the given timestamp is + * older than the first commit timestamp. + */ + if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT) && + commit_ts < txn->first_commit_timestamp) { + __wt_timestamp_to_string(commit_ts, ts_string[0]); + __wt_timestamp_to_string( + txn->first_commit_timestamp, ts_string[1]); + WT_RET_MSG(session, EINVAL, + "commit timestamp %s older than the first " + "commit timestamp %s for this transaction", + ts_string[0], ts_string[1]); + } + } else { + /* + * For a prepared transaction, the commit timestamp should not + * be less than the prepare timestamp. + */ + if (txn->prepare_timestamp > commit_ts) { + if (!F_ISSET(txn, WT_TXN_TS_ROUND_PREPARED)) { + __wt_timestamp_to_string( + commit_ts, ts_string[0]); + __wt_timestamp_to_string( + txn->prepare_timestamp, ts_string[1]); + WT_RET_MSG(session, EINVAL, + "commit timestamp %s is less than the " + "prepare timestamp %s for this transaction", + ts_string[0], ts_string[1]); + } + commit_ts = txn->prepare_timestamp; + } } - + WT_ASSERT(session, txn->durable_timestamp == WT_TS_NONE || + txn->durable_timestamp == txn->commit_timestamp); + txn->durable_timestamp = txn->commit_timestamp = commit_ts; + F_SET(txn, WT_TXN_HAS_TS_COMMIT); return (0); } /* - * __wt_txn_set_timestamp -- - * Parse a request to set a timestamp in a transaction. + * __wt_txn_set_durable_timestamp -- + * Validate the durable timestamp of a transaction. */ int -__wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_txn_set_durable_timestamp( + WT_SESSION_IMPL *session, wt_timestamp_t durable_ts) { - WT_CONFIG_ITEM cval; - WT_DECL_RET; - WT_TXN *txn; - wt_timestamp_t ts; - bool prepare, prepare_allowed; + WT_TXN *txn = &session->txn; + WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; + wt_timestamp_t oldest_ts, stable_ts; + char ts_string[2][WT_TS_INT_STRING_SIZE]; + bool has_oldest_ts, has_stable_ts; - txn = &session->txn; - prepare_allowed = false; - prepare = F_ISSET(txn, WT_TXN_PREPARE); + /* Added this redundant initialization to circumvent build failure. */ + oldest_ts = stable_ts = 0; - /* Look for a commit timestamp. */ - ret = __wt_config_gets_def(session, cfg, "commit_timestamp", 0, &cval); - WT_RET_NOTFOUND_OK(ret); - if (ret == 0 && cval.len != 0) { - WT_TRET(__wt_txn_context_check(session, true)); - WT_RET(__wt_txn_parse_timestamp(session, "commit", &ts, &cval)); - /* - * For prepared transactions, commit timestamp can be earlier - * than stable timestamp. - */ - if (prepare) - WT_RET(__wt_txn_commit_timestamp_validate( - session, "commit", ts, &cval, false)); - else - WT_RET(__wt_txn_commit_timestamp_validate( - session, "commit", ts, &cval, true)); - txn->commit_timestamp = ts; - __wt_txn_set_commit_timestamp(session); - txn->durable_timestamp = txn->commit_timestamp; - prepare_allowed = true; - } + if (!F_ISSET(txn, WT_TXN_PREPARE)) + WT_RET_MSG(session, EINVAL, + "durable timestamp should not be specified for " + "non-prepared transaction"); - /* If we have a prepared transaction, look for a durable timestamp. */ - if (prepare) { - ret = __wt_config_gets_def( - session, cfg, "durable_timestamp", 0, &cval); - WT_RET_NOTFOUND_OK(ret); - if (ret == 0 && cval.len != 0) { - WT_TRET(__wt_txn_context_check(session, true)); - WT_RET(__wt_txn_parse_timestamp( - session, "durable", &ts, &cval)); - txn->durable_timestamp = ts; - F_SET(txn, WT_TXN_HAS_TS_DURABLE); - prepare_allowed = true; - } - } + if (!F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) + WT_RET_MSG(session, EINVAL, + "commit timestamp is needed before the durable timestamp"); /* - * We copy the commit_timestamp as durable_timestamp, hence validation - * is required. + * Compare against the oldest and the stable timestamp. Return an error + * if the given timestamp is less than oldest and/or stable timestamp. */ - if (ret == 0 && cval.len != 0) - WT_RET(__wt_txn_commit_timestamp_validate( - session, "durable", txn->durable_timestamp, &cval, true)); + has_oldest_ts = txn_global->has_oldest_timestamp; + if (has_oldest_ts) + oldest_ts = txn_global->oldest_timestamp; + has_stable_ts = txn_global->has_stable_timestamp; + if (has_stable_ts) + stable_ts = txn_global->stable_timestamp; + /* - * We allow setting the commit timestamp and durable timestamp after a - * prepare but no other timestamp. + * For a non-prepared transactions the commit timestamp should + * not be less than the stable timestamp. */ - if (!prepare_allowed) - WT_RET(__wt_txn_context_prepare_check(session)); + if (has_oldest_ts && durable_ts < oldest_ts) { + __wt_timestamp_to_string(durable_ts, ts_string[0]); + __wt_timestamp_to_string(oldest_ts, ts_string[1]); + WT_RET_MSG(session, EINVAL, + "durable timestamp %s is less than the oldest timestamp %s", + ts_string[0], ts_string[1]); + } - /* Look for a read timestamp. */ - WT_RET(__wt_txn_parse_read_timestamp(session, cfg)); + if (has_stable_ts && durable_ts < stable_ts) { + __wt_timestamp_to_string(durable_ts, ts_string[0]); + __wt_timestamp_to_string(oldest_ts, ts_string[1]); + WT_RET_MSG(session, EINVAL, + "durable timestamp %s is less than the stable timestamp %s", + ts_string[0], ts_string[1]); + } + + /* Check if the durable timestamp is less than the commit timestamp. */ + if (durable_ts < txn->commit_timestamp) { + __wt_timestamp_to_string(durable_ts, ts_string[0]); + __wt_timestamp_to_string(txn->commit_timestamp, ts_string[1]); + WT_RET_MSG(session, EINVAL, + "durable timestamp %s is less than the commit timestamp %s " + "for this transaction", + ts_string[0], ts_string[1]); + } + txn->durable_timestamp = durable_ts; + F_SET(txn, WT_TXN_HAS_TS_DURABLE); return (0); } /* - * __wt_txn_parse_prepare_timestamp -- - * Parse a request to set a transaction's prepare_timestamp. + * __wt_txn_set_prepare_timestamp -- + * Validate and set the prepare timestamp of a transaction. */ int -__wt_txn_parse_prepare_timestamp( - WT_SESSION_IMPL *session, const char *cfg[]) +__wt_txn_set_prepare_timestamp( + WT_SESSION_IMPL *session, wt_timestamp_t prepare_ts) { - WT_CONFIG_ITEM cval; - WT_TXN *prev, *txn; - WT_TXN_GLOBAL *txn_global; - wt_timestamp_t oldest_ts, timestamp, tmp_timestamp; + WT_TXN *prev, *txn = &session->txn; + WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; + wt_timestamp_t oldest_ts, tmp_timestamp; char ts_string[2][WT_TS_INT_STRING_SIZE]; - txn = &session->txn; - txn_global = &S2C(session)->txn_global; + WT_RET(__wt_txn_context_prepare_check(session)); - WT_RET(__wt_config_gets_def(session, - cfg, "prepare_timestamp", 0, &cval)); - if (cval.len == 0) - WT_RET_MSG(session, EINVAL, "prepare timestamp is required"); + if (F_ISSET(txn, WT_TXN_HAS_TS_PREPARE)) + WT_RET_MSG(session, EINVAL, "prepare timestamp is already set"); - if (F_ISSET(&session->txn, WT_TXN_HAS_TS_COMMIT)) + if (F_ISSET(txn, WT_TXN_HAS_TS_COMMIT)) WT_RET_MSG(session, EINVAL, "commit timestamp " - "should not have been set before prepare transaction"); + "should not have been set before the prepare timestamp"); - WT_RET(__wt_txn_parse_timestamp(session, "prepare", ×tamp, &cval)); __wt_readlock(session, &txn_global->read_timestamp_rwlock); oldest_ts = txn_global->oldest_timestamp; /* @@ -804,15 +810,15 @@ __wt_txn_parse_prepare_timestamp( continue; } - if (tmp_timestamp >= timestamp) { + if (tmp_timestamp >= prepare_ts) { __wt_readunlock(session, &txn_global->read_timestamp_rwlock); - __wt_timestamp_to_string( - tmp_timestamp, ts_string[0]); + __wt_timestamp_to_string(prepare_ts, ts_string[0]); + __wt_timestamp_to_string(tmp_timestamp, ts_string[1]); WT_RET_MSG(session, EINVAL, - "prepare timestamp %.*s must be greater than the " + "prepare timestamp %s must be greater than the " "latest active read timestamp %s ", - (int)cval.len, cval.str, ts_string[0]); + ts_string[0], ts_string[1]); } break; } @@ -823,7 +829,7 @@ __wt_txn_parse_prepare_timestamp( * Check whether the prepare timestamp is less than the oldest * timestamp. */ - if (timestamp < oldest_ts) { + if (prepare_ts < oldest_ts) { /* * Check whether the prepare timestamp needs to be rounded up to * the oldest timestamp. @@ -838,140 +844,185 @@ __wt_txn_parse_prepare_timestamp( if (WT_VERBOSE_ISSET(session, WT_VERB_TIMESTAMP)) { __wt_timestamp_to_string( - timestamp, ts_string[0]); + prepare_ts, ts_string[0]); __wt_timestamp_to_string( oldest_ts, ts_string[1]); __wt_verbose(session, WT_VERB_TIMESTAMP, "prepare timestamp %s rounded to oldest " "timestamp %s", ts_string[0], ts_string[1]); } - timestamp = oldest_ts; + prepare_ts = oldest_ts; } else { - __wt_timestamp_to_string( - oldest_ts, ts_string[0]); + __wt_timestamp_to_string(prepare_ts, ts_string[0]); + __wt_timestamp_to_string(oldest_ts, ts_string[0]); WT_RET_MSG(session, EINVAL, - "prepare timestamp %.*s is older than the oldest " - "timestamp %s ", (int)cval.len, cval.str, - ts_string[0]); + "prepare timestamp %s is older than the oldest " + "timestamp %s ", ts_string[0], ts_string[1]); } } - txn->prepare_timestamp = timestamp; + txn->prepare_timestamp = prepare_ts; + F_SET(txn, WT_TXN_HAS_TS_PREPARE); return (0); } /* - * __wt_txn_parse_read_timestamp -- + * __wt_txn_set_read_timestamp -- * Parse a request to set a transaction's read_timestamp. */ int -__wt_txn_parse_read_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_txn_set_read_timestamp( + WT_SESSION_IMPL *session, wt_timestamp_t read_ts) { - WT_CONFIG_ITEM cval; - WT_TXN *txn; - WT_TXN_GLOBAL *txn_global; - wt_timestamp_t ts, ts_oldest; + WT_TXN *txn = &session->txn; + WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global; + wt_timestamp_t ts_oldest; char ts_string[2][WT_TS_INT_STRING_SIZE]; - bool round_to_oldest; + bool roundup_to_oldest; - txn = &session->txn; + WT_RET(__wt_txn_context_prepare_check(session)); - WT_RET(__wt_config_gets_def(session, cfg, "read_timestamp", 0, &cval)); - if (cval.len > 0) { - txn_global = &S2C(session)->txn_global; - WT_RET(__wt_txn_parse_timestamp(session, "read", &ts, &cval)); + /* Read timestamps imply / require snapshot isolation. */ + if (!F_ISSET(txn, WT_TXN_RUNNING)) + txn->isolation = WT_ISO_SNAPSHOT; + else if (txn->isolation != WT_ISO_SNAPSHOT) + WT_RET_MSG(session, EINVAL, "setting a read_timestamp" + " requires a transaction running at snapshot" + " isolation"); - /* Read timestamps imply / require snapshot isolation. */ - if (!F_ISSET(txn, WT_TXN_RUNNING)) - txn->isolation = WT_ISO_SNAPSHOT; - else if (txn->isolation != WT_ISO_SNAPSHOT) - WT_RET_MSG(session, EINVAL, "setting a read_timestamp" - " requires a transaction running at snapshot" - " isolation"); + /* Read timestamps can't change once set. */ + if (F_ISSET(txn, WT_TXN_HAS_TS_READ)) + WT_RET_MSG(session, EINVAL, "a read_timestamp" + " may only be set once per transaction"); - /* Read timestamps can't change once set. */ - if (F_ISSET(txn, WT_TXN_HAS_TS_READ)) - WT_RET_MSG(session, EINVAL, "a read_timestamp" - " may only be set once per transaction"); + /* + * The read timestamp could be rounded to the oldest timestamp. + */ + roundup_to_oldest = F_ISSET(txn, WT_TXN_TS_ROUND_READ); + /* + * This code is not using the timestamp validate function to + * avoid a race between checking and setting transaction + * timestamp. + */ + __wt_readlock(session, &txn_global->rwlock); + ts_oldest = txn_global->oldest_timestamp; + if (read_ts < ts_oldest) { /* - * Read the configuration here to reduce the span of the - * critical section. + * If given read timestamp is earlier than oldest + * timestamp then round the read timestamp to + * oldest timestamp. */ - WT_RET(__wt_config_gets_def(session, - cfg, "round_to_oldest", 0, &cval)); - round_to_oldest = cval.val; + if (roundup_to_oldest) + txn->read_timestamp = ts_oldest; + else { + __wt_readunlock(session, &txn_global->rwlock); + __wt_timestamp_to_string(read_ts, ts_string[0]); + __wt_timestamp_to_string(ts_oldest, ts_string[1]); + WT_RET_MSG(session, EINVAL, "read timestamp " + "%s less than the oldest timestamp %s", + ts_string[0], ts_string[1]); + } + } else { + txn->read_timestamp = read_ts; /* - * The read timestamp could be rounded to the oldest timestamp. + * Reset to avoid a verbose message as read + * timestamp is not rounded to oldest timestamp. */ - if (F_ISSET(txn, WT_TXN_TS_ROUND_READ)) - round_to_oldest = true; + roundup_to_oldest = false; + } + + __wt_txn_publish_read_timestamp(session); + __wt_readunlock(session, &txn_global->rwlock); + if (roundup_to_oldest && WT_VERBOSE_ISSET(session, WT_VERB_TIMESTAMP)) { /* - * This code is not using the timestamp validate function to - * avoid a race between checking and setting transaction - * timestamp. + * This message is generated here to reduce the span of + * critical section. */ - __wt_readlock(session, &txn_global->rwlock); - ts_oldest = txn_global->oldest_timestamp; - if (ts < ts_oldest) { - /* - * If given read timestamp is earlier than oldest - * timestamp then round the read timestamp to - * oldest timestamp. - */ - if (round_to_oldest) - txn->read_timestamp = ts_oldest; - else { - __wt_readunlock(session, &txn_global->rwlock); - __wt_timestamp_to_string(ts, ts_string[0]); - __wt_timestamp_to_string( - ts_oldest, ts_string[1]); - WT_RET_MSG(session, EINVAL, "read timestamp " - "%s older than oldest timestamp %s", - ts_string[0], ts_string[1]); - } - } else { - txn->read_timestamp = ts; - /* - * Reset to avoid a verbose message as read - * timestamp is not rounded to oldest timestamp. - */ - round_to_oldest = false; - } + __wt_timestamp_to_string(read_ts, ts_string[0]); + __wt_timestamp_to_string(ts_oldest, ts_string[1]); + __wt_verbose(session, WT_VERB_TIMESTAMP, "read " + "timestamp %s : rounded to oldest timestamp %s", + ts_string[0], ts_string[1]); + } - __wt_txn_set_read_timestamp(session); - __wt_readunlock(session, &txn_global->rwlock); - if (round_to_oldest && - WT_VERBOSE_ISSET(session, WT_VERB_TIMESTAMP)) { - /* - * This message is generated here to reduce the span of - * critical section. - */ - __wt_timestamp_to_string(ts, ts_string[0]); - __wt_timestamp_to_string(ts_oldest, ts_string[1]); - __wt_verbose(session, WT_VERB_TIMESTAMP, "Read " - "timestamp %s : Rounded to oldest timestamp %s", - ts_string[0], ts_string[1]); - } + /* + * If we already have a snapshot, it may be too early to match + * the timestamp (including the one we just read, if rounding + * to oldest). Get a new one. + */ + if (F_ISSET(txn, WT_TXN_RUNNING)) + __wt_txn_get_snapshot(session); - /* - * If we already have a snapshot, it may be too early to match - * the timestamp (including the one we just read, if rounding - * to oldest). Get a new one. - */ - if (F_ISSET(txn, WT_TXN_RUNNING)) - __wt_txn_get_snapshot(session); + return (0); +} + +/* + * __wt_txn_set_timestamp -- + * Parse a request to set a timestamp in a transaction. + */ +int +__wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONFIG_ITEM cval; + WT_DECL_RET; + WT_TXN *txn = &session->txn; + wt_timestamp_t ts; + + WT_TRET(__wt_txn_context_check(session, true)); + + /* Look for round_to_oldest configuration. */ + ret = __wt_config_gets_def(session, cfg, "round_to_oldest", 0, &cval); + if (cval.val) + F_SET(txn, WT_TXN_TS_ROUND_READ); + + /* Look for a commit timestamp. */ + ret = __wt_config_gets_def(session, cfg, "commit_timestamp", 0, &cval); + WT_RET_NOTFOUND_OK(ret); + if (ret == 0 && cval.len != 0) { + WT_RET(__wt_txn_parse_timestamp(session, "commit", &ts, &cval)); + WT_RET(__wt_txn_set_commit_timestamp(session, ts)); + __wt_txn_publish_commit_timestamp(session); + } + + /* + * Look for a durable timestamp. Durable timestamp should be set only + * after setting the commit timestamp. + */ + ret = __wt_config_gets_def( + session, cfg, "durable_timestamp", 0, &cval); + WT_RET_NOTFOUND_OK(ret); + if (ret == 0 && cval.len != 0) { + WT_RET(__wt_txn_parse_timestamp( + session, "durable", &ts, &cval)); + WT_RET(__wt_txn_set_durable_timestamp(session, ts)); + } + + /* Look for a read timestamp. */ + WT_RET(__wt_config_gets_def(session, cfg, "read_timestamp", 0, &cval)); + if (ret == 0 && cval.len != 0) { + WT_RET(__wt_txn_parse_timestamp(session, "read", &ts, &cval)); + WT_RET(__wt_txn_set_read_timestamp(session, ts)); + } + + /* Look for a prepare timestamp. */ + WT_RET(__wt_config_gets_def(session, + cfg, "prepare_timestamp", 0, &cval)); + if (ret == 0 && cval.len != 0) { + WT_RET(__wt_txn_parse_timestamp( + session, "prepare", &ts, &cval)); + WT_RET(__wt_txn_set_prepare_timestamp(session, ts)); } return (0); } /* - * __wt_txn_set_commit_timestamp -- + * __wt_txn_publish_commit_timestamp -- * Publish a transaction's commit timestamp. */ void -__wt_txn_set_commit_timestamp(WT_SESSION_IMPL *session) +__wt_txn_publish_commit_timestamp(WT_SESSION_IMPL *session) { WT_TXN *qtxn, *txn, *txn_tmp; WT_TXN_GLOBAL *txn_global; @@ -1086,11 +1137,11 @@ __wt_txn_clear_commit_timestamp(WT_SESSION_IMPL *session) } /* - * __wt_txn_set_read_timestamp -- + * __wt_txn_publish_read_timestamp -- * Publish a transaction's read timestamp. */ void -__wt_txn_set_read_timestamp(WT_SESSION_IMPL *session) +__wt_txn_publish_read_timestamp(WT_SESSION_IMPL *session) { WT_TXN *qtxn, *txn, *txn_tmp; WT_TXN_GLOBAL *txn_global; diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index 6942f44e581..14223227f12 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -784,15 +784,32 @@ thread_run(void *arg) session, tscfg)); if (i % PREPARE_YIELD == 0) __wt_yield(); - } - testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), - "commit_timestamp=%" PRIx64, stable_ts)); + + testutil_check(__wt_snprintf( + tscfg, sizeof(tscfg), + "commit_timestamp=%" PRIx64 + ",durable_timestamp=%" PRIx64, + stable_ts, stable_ts)); + } else + testutil_check(__wt_snprintf( + tscfg, sizeof(tscfg), + "commit_timestamp=%" PRIx64, stable_ts)); + testutil_check( session->commit_transaction(session, tscfg)); - if (use_prep) + if (use_prep) { + /* + * Durable timestamp should not be passed as + * oplog transaction is a non-prepared + * transaction. + */ + testutil_check(__wt_snprintf( + tscfg, sizeof(tscfg), + "commit_timestamp=%" PRIx64, stable_ts)); testutil_check( oplog_session->commit_transaction( oplog_session, tscfg)); + } /* * Update the thread's last-committed timestamp. * Don't let the compiler re-order this statement, diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 3f52bbb9c2d..fb877fed86a 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -370,10 +370,16 @@ thread_run(void *arg) prepared_session, tscfg)); if (i % PREPARE_YIELD == 0) __wt_yield(); - } - testutil_check( - __wt_snprintf(tscfg, sizeof(tscfg), - "commit_timestamp=%" PRIx64, active_ts)); + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), + "commit_timestamp=%" PRIx64 + ",durable_timestamp=%" PRIx64, + active_ts, active_ts)); + } else + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), + "commit_timestamp=%" PRIx64, active_ts)); + testutil_check( prepared_session->commit_transaction( prepared_session, tscfg)); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index b370e8507c3..00869b45294 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -16,6 +16,30 @@ functions: remote_file: wiredtiger/${build_variant}/${revision}/artifacts/${build_id}.tgz bucket: build_external extract_to: wiredtiger + "fetch artifacts from little-endian" : + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: wiredtiger/little-endian/${revision}/artifacts/WT_TEST.tgz + bucket: build_external + local_file: WT_TEST-little-endian.tgz + - command: archive.targz_extract + params: + path: "WT_TEST-little-endian.tgz" + destination: "wiredtiger/build_posix/test/format" + "fetch artifacts from big-endian" : + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: wiredtiger/big-endian/${revision}/artifacts/WT_TEST.tgz + bucket: build_external + local_file: WT_TEST-big-endian.tgz + - command: archive.targz_extract + params: + path: "WT_TEST-big-endian.tgz" + destination: "wiredtiger/build_posix/test/format" "fetch mongo-tests repo" : command: shell.exec params: @@ -59,7 +83,7 @@ post: set -o errexit set -o verbose cd wiredtiger - tar cvfz ../wiredtiger.tgz . + tar cfz ../wiredtiger.tgz . cd .. - command: s3.put params: @@ -976,6 +1000,128 @@ tasks: set -o verbose test/evergreen/compatibility_test_for_mongodb_releases.sh + - name: generate-datafile-little-endian + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "compile wiredtiger" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/format" + script: | + set -o errexit + set -o verbose + for i in $(seq 10) + do + ./t -1 -h "WT_TEST.$i" -c $(pwd)/../../../test/format/CONFIG.endian + done + # Archive the WT_TEST directories which include the generated wt data files + tar -zcvf WT_TEST.tgz WT_TEST* + shell: bash + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_file: wiredtiger/build_posix/test/format/WT_TEST.tgz + bucket: build_external + permissions: public-read + content_type: application/tar + display_name: WT_TEST + remote_file: wiredtiger/little-endian/${revision}/artifacts/WT_TEST.tgz + + - name: verify-datafile-little-endian + depends_on: + - name: compile + - name: generate-datafile-little-endian + commands: + - func: "fetch artifacts" + - func: "fetch artifacts from little-endian" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + ./test/evergreen/verify_wt_datafiles.sh 2>&1 + + - name: verify-datafile-from-little-endian + depends_on: + - name: compile + - name: generate-datafile-little-endian + variant: little-endian + commands: + - func: "fetch artifacts" + - func: "fetch artifacts from little-endian" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + ./test/evergreen/verify_wt_datafiles.sh 2>&1 + + - name: generate-datafile-big-endian + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "compile wiredtiger" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/format" + script: | + set -o errexit + set -o verbose + for i in $(seq 10) + do + ./t -1 -h "WT_TEST.$i" -c $(pwd)/../../../test/format/CONFIG.endian + done + # Archive the WT_TEST directories which include the generated wt data files + tar -zcvf WT_TEST.tgz WT_TEST* + shell: bash + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_file: wiredtiger/build_posix/test/format/WT_TEST.tgz + bucket: build_external + permissions: public-read + content_type: application/tar + display_name: WT_TEST + remote_file: wiredtiger/big-endian/${revision}/artifacts/WT_TEST.tgz + + - name: verify-datafile-big-endian + depends_on: + - name: compile + - name: generate-datafile-big-endian + commands: + - func: "fetch artifacts" + - func: "fetch artifacts from big-endian" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + ./test/evergreen/verify_wt_datafiles.sh 2>&1 + + - name: verify-datafile-from-big-endian + depends_on: + - name: compile + - name: generate-datafile-big-endian + variant: big-endian + commands: + - func: "fetch artifacts" + - func: "fetch artifacts from big-endian" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + ./test/evergreen/verify_wt_datafiles.sh 2>&1 + buildvariants: - name: ubuntu1404 @@ -1095,3 +1241,34 @@ buildvariants: - name: unit-test - name: fops +- name: little-endian + display_name: Little-endian (x86) + run_on: + - ubuntu1404-test + batchtime: 1440 # 1 day + expansions: + smp_command: -j $(grep -c ^processor /proc/cpuinfo) + configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs + tasks: + - name: compile + - name: generate-datafile-little-endian + - name: verify-datafile-little-endian + - name: verify-datafile-from-big-endian + +- name: big-endian + display_name: Big-endian (s390x/zSeries) + modules: + - enterprise + run_on: + - ubuntu1604-zseries-small + batchtime: 1440 # 1 day + expansions: + smp_command: -j $(grep -c ^processor /proc/cpuinfo) + configure_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH + test_env_vars: PATH=/opt/mongodbtoolchain/v3/bin:$PATH LD_LIBRARY_PATH=$(pwd)/.libs + tasks: + - name: compile + - name: generate-datafile-big-endian + - name: verify-datafile-big-endian + - name: verify-datafile-from-little-endian diff --git a/src/third_party/wiredtiger/test/evergreen/verify_wt_datafiles.sh b/src/third_party/wiredtiger/test/evergreen/verify_wt_datafiles.sh new file mode 100755 index 00000000000..4c7b56d34ea --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/verify_wt_datafiles.sh @@ -0,0 +1,105 @@ +#! /usr/bin/env bash +# +# This script is used to list and dump WiredTiger data files ('WiredTiger.wt') generated by test/format program. +# +# Supported return values: +# 0: list & verify & dump are successful for all data files +# 1: list is not successful for some data file +# 2: verify is not successful for some table object +# 3: dump is not successful for some table object +# 4: No table is dumped out from all data files +# 5: Command argument setting is wrong +# 6: 't' or 'wt' binary file does not exist +# 7: 'WT_TEST' directories do not exist + +set -eu + +verbose=false + +if [ $# -gt 1 ]; then + echo "Usage: $0 [-v]" + exit 5 +elif [ $# -eq 1 ]; then + [ "$1" == "-v" ] && verbose=true || verbose=false +fi + +# Switch to the Git repo toplevel directory +cd $(git rev-parse --show-toplevel) + +# Walk into the test/format directory in which data files are generated +cd build_posix/test/format + +# Check the existence of 'WT_TEST' directories +num_dirs=$(find . -type d -name 'WT_TEST.[0-9]*' | wc -l) +if [ "${num_dirs}" -eq "0" ]; then + echo "test/format 'WT_TEST' directories do not exist, exiting ..." + exit 7 +fi + +# Check the existence of 'wt' binary +wt_binary="../../.libs/wt" +if [ ! -x "${wt_binary}" ]; then + echo "'wt' binary does not exist, exiting ..." + exit 6 +fi + +# Check the existence of test/format 't' binary +if [ ! -x "t" ]; then + echo "test/format 't' binary does not exist, exiting ..." + exit 6 +fi + +lib_dir=$(dirname "${wt_binary}") +num_tables_verified=0 + +# Work out the list of directories that include wt data files +dirs_include_datafile=$(find ./WT_TEST.[0-9]* -type f -name WiredTiger.wt -print0 | xargs -0 dirname) +echo -e "\n[dirs_include_datafile]:\n${dirs_include_datafile}\n" + +# Loop through each data file under the TEST_DIR +IFS=$'\n' +for d in ${dirs_include_datafile} +do + echo "${d}" + + tables=$(LD_LIBRARY_PATH="${lib_dir}" ${wt_binary} -h "${d}" list) + rc=$? + + if [ "$rc" -ne "0" ]; then + echo "Failed to list '${d}' directory, exiting ..." + exit 1 + fi + + # Loop through each table object in the data file + for t in ${tables} + do + echo ${t} + LD_LIBRARY_PATH="${lib_dir}" ${wt_binary} -h ${d} verify ${t} + if [ "$?" -ne "0" ]; then + echo "Failed to verify '${t}' table under '${d}' directory, exiting ..." + exit 2 + fi + + dump=$(LD_LIBRARY_PATH="${lib_dir}" ${wt_binary} -h ${d} dump ${t}) + if [ "$?" -ne "0" ]; then + echo "Failed to dump '${t}' table under '${d}' directory, exiting ..." + exit 3 + fi + + # Print out the table dump if verbose flag is on + [[ "${verbose}" == true ]] && echo ${dump} + + # Table verification is successful, increment the counter + let "num_tables_verified+=1" + echo "num_tables_verified = ${num_tables_verified}" + done +done + +if [ "${num_tables_verified}" -eq 0 ]; then + echo "No table is verified from all data files, something could have gone wrong ..." + exit 4 +fi + +# If reaching here, the testing result is positive +echo -e "\nAll data files under 'WT_TEST' directories are verified successfully!" +exit 0 diff --git a/src/third_party/wiredtiger/test/format/CONFIG.endian b/src/third_party/wiredtiger/test/format/CONFIG.endian new file mode 100644 index 00000000000..d1fd18a677b --- /dev/null +++ b/src/third_party/wiredtiger/test/format/CONFIG.endian @@ -0,0 +1,3 @@ +cache_minimum=20 +rows=1000000 +timer=4 diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 75c1add2447..7e95e696550 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -290,6 +290,8 @@ typedef struct { WT_RAND_STATE rnd; /* thread RNG state */ + bool prepare_txn; /* is prepare transaction */ + volatile bool quit; /* thread should quit */ uint64_t ops; /* total operations */ diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 57cbd7168db..663aaad4677 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -639,9 +639,18 @@ commit_transaction(TINFO *tinfo, WT_SESSION *session) buf, sizeof(buf), "commit_timestamp=%" PRIx64, ts)); testutil_check(session->timestamp_transaction(session, buf)); + if (tinfo->prepare_txn) { + testutil_check(__wt_snprintf(buf, sizeof(buf), + "durable_timestamp=%" PRIx64, ts)); + testutil_check( + session->timestamp_transaction(session, buf)); + } + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); } testutil_check(session->commit_transaction(session, NULL)); + + tinfo->prepare_txn = false; } /* @@ -654,6 +663,8 @@ rollback_transaction(TINFO *tinfo, WT_SESSION *session) ++tinfo->rollback; testutil_check(session->rollback_transaction(session, NULL)); + + tinfo->prepare_txn = false; } /* @@ -687,6 +698,8 @@ prepare_transaction(TINFO *tinfo, WT_SESSION *session) ret = session->prepare_transaction(session, buf); testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + + tinfo->prepare_txn = true; return (ret); } diff --git a/src/third_party/wiredtiger/test/suite/test_assert05.py b/src/third_party/wiredtiger/test/suite/test_assert05.py index b709e8df8dd..39c79054baf 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert05.py +++ b/src/third_party/wiredtiger/test/suite/test_assert05.py @@ -85,15 +85,17 @@ class test_assert05(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() c[key] = val - self.session.prepare_transaction( - 'prepare_timestamp=' + timestamp_str(self.count)) + if (use_ts == 'always'): + self.session.prepare_transaction( + 'prepare_timestamp=' + timestamp_str(self.count)) + self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(self.count)) # All settings other than always should commit successfully if (use_ts != 'always'): self.session.commit_transaction() else: - msg = "/none set on this transaction/" + msg = "/durable_timestamp is required for a prepared/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:self.assertEquals(self.session.commit_transaction(), 0), msg) diff --git a/src/third_party/wiredtiger/test/suite/test_assert06.py b/src/third_party/wiredtiger/test/suite/test_assert06.py index 828968441a9..17b38fb0f03 100644 --- a/src/third_party/wiredtiger/test/suite/test_assert06.py +++ b/src/third_party/wiredtiger/test/suite/test_assert06.py @@ -368,8 +368,9 @@ class test_assert06(wttest.WiredTigerTestCase, suite_subprocess): 'prepare_timestamp=' + timestamp_str(22)) self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(22)) - self.session.commit_transaction('durable_timestamp=' + - timestamp_str(22)) + self.session.timestamp_transaction( + 'durable_timestamp=' + timestamp_str(22)) + self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_compat01.py b/src/third_party/wiredtiger/test/suite/test_compat01.py index 9670a4cc137..faf5ed1673f 100644 --- a/src/third_party/wiredtiger/test/suite/test_compat01.py +++ b/src/third_party/wiredtiger/test/suite/test_compat01.py @@ -189,12 +189,20 @@ class test_reconfig_fail(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, uri, 100, key_format='S') ds.populate() + # Reconfigure to an older version. + compat_str = 'compatibility=(release="2.6")' + self.conn.reconfigure(compat_str) + self.session.begin_transaction("isolation=snapshot") c = self.session.open_cursor(uri, None) c.set_key(ds.key(20)) c.set_value("abcde") self.assertEquals(c.update(), 0) + # Make sure we can reconfigure unrelated things while downgraded + # and we have an active transaction. + self.conn.reconfigure("cache_size=100M") + compat_str = 'compatibility=(release="3.0.0")' msg = '/system must be quiescent/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py index e42d8458fe5..e54bd3deaf2 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py @@ -87,7 +87,9 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase): self.assertEquals(cursor.next(), 0) session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.commit_transaction('commit_timestamp=' + timestamp_str(200) + ',durable_timestamp=' + timestamp_str(220)) + session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. @@ -133,7 +135,9 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase): # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. - session.commit_transaction('commit_timestamp=' + timestamp_str(240) + ',durable_timestamp=' + timestamp_str(300)) + session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) + session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + session.commit_transaction() # Checkpoint so that first update value will be visible and durable, # but second update value will be only visible but not durable. diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py index 6f2d9b1cb97..c62372629cb 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py @@ -87,7 +87,9 @@ class test_durable_ts01(wttest.WiredTigerTestCase): self.assertEquals(cursor.next(), 0) session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.commit_transaction('commit_timestamp=' + timestamp_str(200) + ',durable_timestamp=' + timestamp_str(220)) + session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. @@ -133,7 +135,9 @@ class test_durable_ts01(wttest.WiredTigerTestCase): # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. - session.commit_transaction('commit_timestamp=' + timestamp_str(240) + ',durable_timestamp=' + timestamp_str(300)) + session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) + session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + session.commit_transaction() # Checkpoint so that first update value will be visible and durable, # but second update value will be only visible but not durable. diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py index 8a475762d9f..feb238aa268 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py @@ -87,7 +87,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): self.assertEquals(cursor.next(), 0) session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - msg = "/older than the first commit timestamp/" + msg = "/is less than the commit timestamp/" # Check for error when commit timestamp > durable timestamp. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: session.commit_transaction('commit_timestamp=' + timestamp_str(200) + ',durable_timestamp=' + timestamp_str(180)), msg) @@ -107,7 +107,7 @@ class test_durable_ts03(wttest.WiredTigerTestCase): session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - msg = "/older than stable timestamp/" + msg = "/is less than the stable timestamp/" # Check that error is returned when durable timestamp < stable timestamp. self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: session.commit_transaction('commit_timestamp=' + timestamp_str(200) + ',durable_timestamp=' + timestamp_str(240)), msg) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index a43bc786e36..e606a223953 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -72,8 +72,9 @@ class test_durable_ts03(wttest.WiredTigerTestCase): session.begin_transaction() cursor[i] = valueB session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) - session.commit_transaction('commit_timestamp=' + timestamp_str(200) + \ - ',durable_timestamp=' + timestamp_str(220)) + session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) + session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) + session.commit_transaction() # Check the checkpoint wrote only the durable updates. cursor2 = self.session.open_cursor( @@ -111,8 +112,9 @@ class test_durable_ts03(wttest.WiredTigerTestCase): session.begin_transaction() cursor[i] = valueC session.prepare_transaction('prepare_timestamp=' + timestamp_str(220)) - session.commit_transaction('commit_timestamp=' + timestamp_str(230) + \ - ',durable_timestamp=' + timestamp_str(240)) + session.timestamp_transaction('commit_timestamp=' + timestamp_str(230)) + session.timestamp_transaction('durable_timestamp=' + timestamp_str(240)) + session.commit_transaction() self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare01.py b/src/third_party/wiredtiger/test/suite/test_prepare01.py index 6d9d0013cad..c23d0a869b2 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare01.py @@ -116,7 +116,9 @@ class test_prepare01(wttest.WiredTigerTestCase): if i > 0 and i % (self.nentries / 37) == 0: self.check(cursor, committed, i) self.session.prepare_transaction("prepare_timestamp=2a") - self.session.commit_transaction("commit_timestamp=3a") + self.session.timestamp_transaction("commit_timestamp=3a") + self.session.timestamp_transaction("durable_timestamp=3a") + self.session.commit_transaction() committed = i self.session.begin_transaction() @@ -132,8 +134,11 @@ class test_prepare01(wttest.WiredTigerTestCase): self.check(cursor, committed, self.nentries) - self.session.prepare_transaction("prepare_timestamp=2a") - self.session.commit_transaction("commit_timestamp=3a") + self.session.timestamp_transaction("prepare_timestamp=2a") + self.session.prepare_transaction() + self.session.timestamp_transaction("commit_timestamp=3a") + self.session.timestamp_transaction("durable_timestamp=3a") + self.session.commit_transaction() self.check(cursor, self.nentries, self.nentries) # Test that read-committed is the default isolation level. @@ -154,7 +159,9 @@ class test_read_committed_default(wttest.WiredTigerTestCase): cursor['key: aaa'] = 'value: aaa' self.session.prepare_transaction("prepare_timestamp=2a") - self.session.commit_transaction("commit_timestamp=3a") + self.session.timestamp_transaction("commit_timestamp=3a") + self.session.timestamp_transaction("durable_timestamp=3a") + self.session.commit_transaction() self.session.begin_transaction() cursor['key: bbb'] = 'value: bbb' @@ -165,13 +172,17 @@ class test_read_committed_default(wttest.WiredTigerTestCase): s.prepare_transaction("prepare_timestamp=4a") # commit timestamp can be same as prepare timestamp - s.commit_transaction("commit_timestamp=4a") + s.timestamp_transaction("commit_timestamp=4a") + s.timestamp_transaction("durable_timestamp=4a") + s.commit_transaction() s.begin_transaction() self.assertEqual(self.cursor_count(cursor), 1) s.prepare_transaction("prepare_timestamp=7a") # commit timestamp can be greater than prepare timestamp - s.commit_transaction("commit_timestamp=8a") + s.timestamp_transaction("commit_timestamp=8a") + s.timestamp_transaction("durable_timestamp=8a") + s.commit_transaction() s.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_prepare02.py b/src/third_party/wiredtiger/test/suite/test_prepare02.py index 688fb04c09f..fc92adaad63 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare02.py @@ -109,7 +109,9 @@ class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() c1 = self.session.open_cursor("table:mytable", None) self.session.prepare_transaction("prepare_timestamp=2a") - self.session.commit_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("durable_timestamp=2b") + self.session.commit_transaction() # Setting commit timestamp via timestamp_transaction after # prepare is also permitted. @@ -117,6 +119,7 @@ class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): c1 = self.session.open_cursor("table:mytable", None) self.session.prepare_transaction("prepare_timestamp=2a") self.session.timestamp_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("durable_timestamp=2b") self.session.commit_transaction() # Rollback after prepare is permitted. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare03.py b/src/third_party/wiredtiger/test/suite/test_prepare03.py index 1d39cbf720c..512efe0c008 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare03.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare03.py @@ -93,7 +93,9 @@ class test_prepare03(wttest.WiredTigerTestCase): self.session.prepare_transaction("prepare_timestamp=2a") self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:cursor.insert(), preparemsg) - self.session.commit_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("durable_timestamp=2b") + self.session.commit_transaction() cursor.insert() # Check next, get_key, get_value operations. @@ -110,7 +112,9 @@ class test_prepare03(wttest.WiredTigerTestCase): lambda:cursor.get_key(), preparemsg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:cursor.get_value(), preparemsg) - self.session.commit_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("durable_timestamp=2b") + self.session.commit_transaction() nextret = cursor.next() if nextret != 0: break @@ -134,7 +138,9 @@ class test_prepare03(wttest.WiredTigerTestCase): self.session.prepare_transaction("prepare_timestamp=2a") self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:cursor.prev(), preparemsg) - self.session.commit_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("durable_timestamp=2b") + self.session.commit_transaction() prevret = cursor.prev() if prevret != 0: break @@ -168,7 +174,9 @@ class test_prepare03(wttest.WiredTigerTestCase): lambda:cursor.reserve(), preparemsg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:cursor.reconfigure(), preparemsg) - self.session.commit_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("durable_timestamp=2b") + self.session.commit_transaction() cursor.search() cursor.set_value(self.genvalue(self.nentries + self.nentries/2)) cursor.update() @@ -180,7 +188,9 @@ class test_prepare03(wttest.WiredTigerTestCase): self.session.prepare_transaction("prepare_timestamp=2a") self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:cursor.search_near(), preparemsg) - self.session.commit_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("commit_timestamp=2b") + self.session.timestamp_transaction("durable_timestamp=2b") + self.session.commit_transaction() # There is a bug with search_near operation when no key is set. # This fix is being tracked in WT-3918. if self.uri == 'lsm': diff --git a/src/third_party/wiredtiger/test/suite/test_prepare04.py b/src/third_party/wiredtiger/test/suite/test_prepare04.py index bf5776934ac..4193c3299b7 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare04.py @@ -113,7 +113,9 @@ class test_prepare04(wttest.WiredTigerTestCase, suite_subprocess): s_other.commit_transaction() #''' - self.session.commit_transaction('commit_timestamp=' + timestamp_str(300)) + self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(300)) + self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) + self.session.commit_transaction() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare05.py b/src/third_party/wiredtiger/test/suite/test_prepare05.py index c236c1f0783..6c1f414ddd7 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare05.py @@ -56,7 +56,9 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): # Check setting the prepare timestamp same as oldest timestamp is valid. self.session.begin_transaction() self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(2)) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(3)) + self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(3)) + self.session.commit_transaction() # In a single transaction it is illegal to set a commit timestamp # before invoking prepare for this transaction. @@ -102,7 +104,7 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( 'commit_timestamp=' + timestamp_str(4)), - "/older than the prepare timestamp/") + "/less than the prepare timestamp/") # It is legal to set a commit timestamp as same as prepare # timestamp. @@ -110,7 +112,9 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): c[1] = 1 self.session.prepare_transaction( 'prepare_timestamp=' + timestamp_str(5)) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(5)) + self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(5)) + self.session.commit_transaction() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py index a4dc312cd7b..68546c4fb6f 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare06.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -47,6 +47,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # It is illegal to set the prepare timestamp older than the oldest # timestamp. self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(20)) + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30)) self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( @@ -58,7 +59,21 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # oldest timestamp is valid with roundup_timestamps settings. self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) + self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(35)) + self.session.commit_transaction() + + # Check setting a prepared transaction timestamps earlier than the + # oldest timestamp is invalid, if durable timestamp is less than the + # stable timestamp. + self.session.begin_transaction('roundup_timestamps=(prepared=true)') + self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(10)) + self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(15)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.timestamp_transaction( + 'durable_timestamp=' + timestamp_str(25)), + "/is less than the stable timestamp/") + self.session.rollback_transaction() # Check the cases with an active reader. # Start a new reader to have an active read timestamp. @@ -67,7 +82,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # It is illegal to set the prepare timestamp as earlier than an active # read timestamp even with roundup_timestamps settings. - self.session.begin_transaction() + self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( 'prepare_timestamp=' + timestamp_str(10)), @@ -76,14 +91,14 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): # It is illegal to set the prepare timestamp the same as an active read # timestamp even with roundup_timestamps settings. - self.session.begin_transaction() + self.session.begin_transaction('roundup_timestamps=(prepared=true)') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction( 'prepare_timestamp=' + timestamp_str(40)), "/must be greater than the latest active read timestamp/") self.session.rollback_transaction() - # It is illegal to set a commit timestamp older than the prepare + # It is illegal to set a commit timestamp less than the prepare # timestamp of a transaction. self.session.begin_transaction() c[1] = 1 @@ -92,7 +107,7 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( 'commit_timestamp=' + timestamp_str(30)), - "/older than the prepare timestamp/") + "/less than the prepare timestamp/") # It is legal to set a commit timestamp older than prepare timestamp of # a transaction with roundup_timestamps settings. @@ -100,7 +115,13 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): c[1] = 1 self.session.prepare_transaction( 'prepare_timestamp=' + timestamp_str(45)) - self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + self.session.timestamp_transaction('commit_timestamp=' + timestamp_str(30)) + #self.session.timestamp_transaction('durable_timestamp=' + timestamp_str(30)) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.timestamp_transaction( + 'durable_timestamp=' + timestamp_str(30)), + "/is less than the commit timestamp/") + self.session.rollback_transaction() s_reader.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py index 213aaad4931..ba53dcbcdac 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py @@ -134,7 +134,9 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.commit_transaction('commit_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(200)) + prep_session.commit_transaction() before_ts_s.commit_transaction() # As read is between(i.e before commit), next is not found. @@ -190,7 +192,9 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.commit_transaction('commit_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) + prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(200)) + prep_session.commit_transaction() # As read is between(i.e before commit), prev is not found. self.assertEquals(between_ts_c.prev(), wiredtiger.WT_NOTFOUND) @@ -248,7 +252,9 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.commit_transaction('commit_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(400)) + prep_session.commit_transaction() # Check to see before cursor still gets the old value. before_ts_c.set_key(ds.key(51)) @@ -310,7 +316,9 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.commit_transaction('commit_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(400)) + prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(400)) + prep_session.commit_transaction() # Check to see before cursor still gets the old value. before_ts_c.set_key(ds.key(1)) @@ -375,7 +383,9 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.commit_transaction('commit_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(600)) + prep_session.commit_transaction() # Check to see before cursor still gets the old value. before_ts_c.set_key(ds.key(51)) @@ -429,7 +439,9 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.commit_transaction('commit_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(600)) + prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(600)) + prep_session.commit_transaction() # Check to see before cursor still gets the old value. before_ts_c.set_key(ds.key(1)) @@ -490,7 +502,9 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.next()) # Commit the prepared transaction. - prep_session.commit_transaction('commit_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(800)) + prep_session.commit_transaction() # Check to see before cursor still gets the old value. before_ts_c.set_key(ds.key(49)) @@ -546,7 +560,9 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): self.assertRaisesException(wiredtiger.WiredTigerError, lambda: cursor.prev()) # Commit the prepared transaction. - prep_session.commit_transaction('commit_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('commit_timestamp=' + timestamp_str(800)) + prep_session.timestamp_transaction('durable_timestamp=' + timestamp_str(800)) + prep_session.commit_transaction() # Check to see before cursor still gets the old value. before_ts_c.set_key(ds.key(3)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py index ae7314a6578..01c59885874 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py @@ -74,7 +74,8 @@ class test_prepare_lookaside02(wttest.WiredTigerTestCase, suite_subprocess): # update the value with in this transaction self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) if self.txn_commit == True: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(101)) + self.session.commit_transaction( + 'commit_timestamp=' + timestamp_str(101) + ',durable_timestamp=' + timestamp_str(101)) else: self.session.rollback_transaction() @@ -95,7 +96,8 @@ class test_prepare_lookaside02(wttest.WiredTigerTestCase, suite_subprocess): c[2] = 2 self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) if self.txn_commit == True: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(201)) + self.session.commit_transaction( + 'commit_timestamp=' + timestamp_str(201) + ',durable_timestamp=' + timestamp_str(201)) else: self.session.rollback_transaction() @@ -121,7 +123,8 @@ class test_prepare_lookaside02(wttest.WiredTigerTestCase, suite_subprocess): c.remove() self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) if self.txn_commit == True: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(301)) + self.session.commit_transaction( + 'commit_timestamp=' + timestamp_str(301) + ',durable_timestamp=' + timestamp_str(301)) else: self.session.rollback_transaction() @@ -158,7 +161,8 @@ class test_prepare_lookaside02(wttest.WiredTigerTestCase, suite_subprocess): cur.remove() self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(400)) if self.txn_commit == True: - self.session.commit_transaction('commit_timestamp=' + timestamp_str(401)) + self.session.commit_transaction( + 'commit_timestamp=' + timestamp_str(401) + ',durable_timestamp=' + timestamp_str(401)) else: self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp09.py b/src/third_party/wiredtiger/test/suite/test_timestamp09.py index ea5a24bf2b5..9e06c38df60 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp09.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp09.py @@ -85,7 +85,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(2)), - '/older than oldest timestamp/') + '/less than the oldest timestamp/') c[2] = 2 self.session.rollback_transaction() @@ -94,7 +94,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( 'commit_timestamp=' + timestamp_str(2)), - '/older than oldest timestamp/') + '/less than the oldest timestamp/') self.session.begin_transaction() c[4] = 4 @@ -135,7 +135,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.timestamp_transaction( 'commit_timestamp=' + timestamp_str(5)), - '/older than stable timestamp/') + '/less than the stable timestamp/') c[5] = 5 self.session.rollback_transaction() @@ -144,7 +144,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( 'commit_timestamp=' + timestamp_str(5)), - '/older than stable timestamp/') + '/less than the stable timestamp/') # When explicitly set, commit timestamp for a transaction can be earlier # than the commit timestamp of an earlier transaction. @@ -167,7 +167,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(6)), - '/older than oldest timestamp/') + '/less than the oldest timestamp/') # c[8] is not visible at read_timestamp < 8 self.session.begin_transaction('read_timestamp=' + timestamp_str(7)) @@ -191,7 +191,7 @@ class test_timestamp09(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.begin_transaction('read_timestamp=' + timestamp_str(4)), - '/older than oldest timestamp/') + '/less than the oldest timestamp/') self.session.begin_transaction('read_timestamp=' + timestamp_str(6)) self.assertTimestampsEqual( self.conn.query_timestamp('get=oldest_reader'), timestamp_str(6)) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp13.py b/src/third_party/wiredtiger/test/suite/test_timestamp13.py index d818ab95e9e..65e0cb7df6f 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp13.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp13.py @@ -132,6 +132,7 @@ class test_timestamp13(wttest.WiredTigerTestCase, suite_subprocess): self.session.query_timestamp('get=prepare'), '10') self.session.timestamp_transaction('commit_timestamp=20') + self.session.timestamp_transaction('durable_timestamp=20') self.assertTimestampsEqual( self.session.query_timestamp('get=prepare'), '10') self.assertTimestampsEqual( diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py index c73cf0ee45d..24b487b193d 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp14.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -307,4 +307,4 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): self.conn.query_timestamp('get=oldest')) if __name__ == '__main__': - wttest.run()
\ No newline at end of file + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp16.py b/src/third_party/wiredtiger/test/suite/test_timestamp16.py index 259503b7d26..375150e4673 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp16.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp16.py @@ -70,4 +70,4 @@ class test_timestamp16(wttest.WiredTigerTestCase, suite_subprocess): self.conn.query_timestamp('get=last_checkpoint')) if __name__ == '__main__': - wttest.run()
\ No newline at end of file + wttest.run() |