diff options
author | Siddhartha Mahajan <siddhartha.mahajan8899@mongodb.com> | 2022-12-09 03:17:16 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-12-09 03:49:43 +0000 |
commit | 64bbe83f61c22ac708749765eae6077615b31e58 (patch) | |
tree | f59326441d3e8156756d43914dc4eaa9d4dacb41 | |
parent | ed586f19b9aa22ad7d0482141b453c107fd20ee6 (diff) | |
download | mongo-64bbe83f61c22ac708749765eae6077615b31e58.tar.gz |
Import wiredtiger: 2862e84453cdb42d0bf31709718fb363d68c3b9f from branch mongodb-master
ref: aed15e961e..2862e84453
for: 6.3.0-rc0
WT-10201 Add RTS dryrun mode
15 files changed, 287 insertions, 63 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 2c3792db158..05bd9a7cfba 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1894,7 +1894,11 @@ methods = { must not be older than the current oldest timestamp. See @ref timestamp_global_api'''), ]), -'WT_CONNECTION.rollback_to_stable' : Method([]), +'WT_CONNECTION.rollback_to_stable' : Method([ + Config('dryrun', 'false', r''' + perform the checks associated with RTS, but don't modify any data.''', + type='boolean'), +]), 'WT_SESSION.reconfigure' : Method(session_config), diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 76d123ae086..8115404882d 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -832,6 +832,7 @@ dmsg doxgen doxygen drealloc +dryrun ds dsb dsbs diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 79a5a3ac8b7..041f9cda337 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "aed15e961e3cc93f7d67231a437203390814093d" + "commit": "2862e84453cdb42d0bf31709718fb363d68c3b9f" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 342b5d17482..e0a58727c05 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -179,6 +179,9 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_WT_CONNECTION_rollback_to_stable[] = { + {"dryrun", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_CHECK confchk_WT_CONNECTION_set_timestamp[] = { {"durable_timestamp", "string", NULL, NULL, NULL, 0}, {"force", "boolean", NULL, NULL, NULL, 0}, {"oldest_timestamp", "string", NULL, NULL, NULL, 0}, @@ -1232,7 +1235,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "tiered_storage=(local_retention=300),timing_stress_for_test=," "verbose=[]", confchk_WT_CONNECTION_reconfigure, 30}, - {"WT_CONNECTION.rollback_to_stable", "", NULL, 0}, {"WT_CONNECTION.set_file_system", "", NULL, 0}, + {"WT_CONNECTION.rollback_to_stable", "dryrun=false", confchk_WT_CONNECTION_rollback_to_stable, 1}, + {"WT_CONNECTION.set_file_system", "", NULL, 0}, {"WT_CONNECTION.set_timestamp", "durable_timestamp=,force=false,oldest_timestamp=," "stable_timestamp=", diff --git a/src/third_party/wiredtiger/src/include/rollback_to_stable.h b/src/third_party/wiredtiger/src/include/rollback_to_stable.h index 5c1fd555e43..773bbd59892 100644 --- a/src/third_party/wiredtiger/src/include/rollback_to_stable.h +++ b/src/third_party/wiredtiger/src/include/rollback_to_stable.h @@ -16,13 +16,30 @@ WT_DECL_VERBOSE_MULTI_CATEGORY(((WT_VERBOSE_CATEGORY[]){WT_VERB_RECOVERY, WT_VERB_RTS})) : \ WT_DECL_VERBOSE_MULTI_CATEGORY(((WT_VERBOSE_CATEGORY[]){WT_VERB_RTS}))) +/* Increment a connection stat if we're not doing a dry run. */ +#define WT_RTS_STAT_CONN_INCR(session, stat) \ + do { \ + if (!S2C(session)->rts->dryrun) \ + WT_STAT_CONN_INCR(session, stat); \ + } while (0) + +/* Increment a connection and data handle stat if we're not doing a dry run. */ +#define WT_RTS_STAT_CONN_DATA_INCR(session, stat) \ + do { \ + if (!S2C(session)->rts->dryrun) \ + WT_STAT_CONN_DATA_INCR(session, stat); \ + } while (0) + /* * WT_ROLLBACK_TO_STABLE -- * Rollback to stable singleton, contains the interface to rollback to stable along * with context used by rollback to stable. */ struct __wt_rollback_to_stable { - /* Methods */ + /* Methods. */ int (*rollback_to_stable_one)(WT_SESSION_IMPL *, const char *, bool *); int (*rollback_to_stable)(WT_SESSION_IMPL *, const char *[], bool); + + /* Configuration. */ + bool dryrun; }; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 7c78ac392c4..e0d3642de95 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2646,7 +2646,10 @@ struct __wt_connection { * @snippet ex_all.c rollback to stable * * @param connection the connection handle - * @configempty{WT_CONNECTION.rollback_to_stable, see dist/api_data.py} + * @configstart{WT_CONNECTION.rollback_to_stable, see dist/api_data.py} + * @config{dryrun, perform the checks associated with RTS\, but don't modify any data., a + * boolean flag; default \c false.} + * @configend * @errors * An error should occur only in the case of a system problem, and an application typically * will retry WT_CONNECTION::rollback_to_stable on error, or fail outright. diff --git a/src/third_party/wiredtiger/src/rollback_to_stable/rts_api.c b/src/third_party/wiredtiger/src/rollback_to_stable/rts_api.c index fc098a348ff..66b3773a57a 100644 --- a/src/third_party/wiredtiger/src/rollback_to_stable/rts_api.c +++ b/src/third_party/wiredtiger/src/rollback_to_stable/rts_api.c @@ -38,9 +38,11 @@ __rollback_to_stable_int(WT_SESSION_IMPL *session, bool no_ckpt) WT_TXN_GLOBAL *txn_global; wt_timestamp_t pinned_timestamp, rollback_timestamp; char ts_string[2][WT_TS_INT_STRING_SIZE]; + bool dryrun; conn = S2C(session); txn_global = &conn->txn_global; + dryrun = conn->rts->dryrun; /* * Rollback to stable should ignore tombstones in the history store since it needs to scan the @@ -95,7 +97,7 @@ __rollback_to_stable_int(WT_SESSION_IMPL *session, bool no_ckpt) * ensure that both in-memory and on-disk versions are the same unless caller requested for no * checkpoint. */ - if (!F_ISSET(conn, WT_CONN_IN_MEMORY) && !no_ckpt) + if (!F_ISSET(conn, WT_CONN_IN_MEMORY) && !no_ckpt && !dryrun) WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); err: @@ -146,15 +148,37 @@ __rollback_to_stable_one(WT_SESSION_IMPL *session, const char *uri, bool *skipp) } /* + * __rollback_to_stable_finalize -- + * Reset a connection's RTS structure in preparation for the next call. + */ +static void +__rollback_to_stable_finalize(WT_ROLLBACK_TO_STABLE *rts) +{ + rts->dryrun = false; +} + +/* * __rollback_to_stable -- * Rollback the database to the stable timestamp. */ static int __rollback_to_stable(WT_SESSION_IMPL *session, const char *cfg[], bool no_ckpt) { + WT_CONFIG_ITEM cval; WT_DECL_RET; + bool dryrun; - WT_UNUSED(cfg); + /* + * Explicit null-check because internal callers (startup/shutdown) do not enter via the API, and + * don't get default values installed in the config string. + */ + dryrun = false; + if (cfg != NULL) { + ret = __wt_config_gets(session, cfg, "dryrun", &cval); + if (ret == 0) + dryrun = cval.val != 0; + WT_RET_NOTFOUND_OK(ret); + } /* * Don't use the connection's default session: we are working on data handles and (a) don't want @@ -165,11 +189,15 @@ __rollback_to_stable(WT_SESSION_IMPL *session, const char *cfg[], bool no_ckpt) WT_RET( __wt_open_internal_session(S2C(session), "txn rollback_to_stable", true, 0, 0, &session)); + S2C(session)->rts->dryrun = dryrun; + WT_STAT_CONN_SET(session, txn_rollback_to_stable_running, 1); WT_WITH_CHECKPOINT_LOCK( session, WT_WITH_SCHEMA_LOCK(session, ret = __rollback_to_stable_int(session, no_ckpt))); WT_STAT_CONN_SET(session, txn_rollback_to_stable_running, 0); + __rollback_to_stable_finalize(S2C(session)->rts); + WT_TRET(__wt_session_close_internal(session)); return (ret); @@ -188,7 +216,10 @@ __wt_rollback_to_stable_init(WT_CONNECTION_IMPL *conn) */ conn->rts = &conn->_rts; - /* Setup function pointers */ + /* Setup function pointers. */ conn->rts->rollback_to_stable = __rollback_to_stable; conn->rts->rollback_to_stable_one = __rollback_to_stable_one; + + /* Setup variables. */ + conn->rts->dryrun = false; } diff --git a/src/third_party/wiredtiger/src/rollback_to_stable/rts_btree.c b/src/third_party/wiredtiger/src/rollback_to_stable/rts_btree.c index 775108596cd..a5450ad9a57 100644 --- a/src/third_party/wiredtiger/src/rollback_to_stable/rts_btree.c +++ b/src/third_party/wiredtiger/src/rollback_to_stable/rts_btree.c @@ -19,8 +19,11 @@ __rts_btree_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *firs { WT_UPDATE *stable_upd, *tombstone, *upd; char ts_string[2][WT_TS_INT_STRING_SIZE]; + bool dryrun; bool txn_id_visible; + dryrun = S2C(session)->rts->dryrun; + stable_upd = tombstone = NULL; txn_id_visible = false; if (stable_update_found != NULL) @@ -54,8 +57,9 @@ __rts_btree_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *firs rollback_timestamp < upd->durable_ts ? "true" : "false", upd->prepare_state, upd->prepare_state == WT_PREPARE_INPROGRESS ? "true" : "false"); - upd->txnid = WT_TXN_ABORTED; - WT_STAT_CONN_INCR(session, txn_rts_upd_aborted); + if (!dryrun) + upd->txnid = WT_TXN_ABORTED; + WT_RTS_STAT_CONN_INCR(session, txn_rts_upd_aborted); } else { /* Valid update is found. */ stable_upd = upd; @@ -97,10 +101,12 @@ __rts_btree_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *firs * Clear the history store flags for the first stable update. Otherwise, it will not be * moved to history store again. */ - if (stable_upd != NULL) - F_CLR(stable_upd, WT_UPDATE_HS | WT_UPDATE_TO_DELETE_FROM_HS); - if (tombstone != NULL) - F_CLR(tombstone, WT_UPDATE_HS | WT_UPDATE_TO_DELETE_FROM_HS); + if (!dryrun) { + if (stable_upd != NULL) + F_CLR(stable_upd, WT_UPDATE_HS | WT_UPDATE_TO_DELETE_FROM_HS); + if (tombstone != NULL) + F_CLR(tombstone, WT_UPDATE_HS | WT_UPDATE_TO_DELETE_FROM_HS); + } } if (stable_update_found != NULL) *stable_update_found = true; @@ -175,6 +181,9 @@ __rts_btree_col_modify(WT_SESSION_IMPL *session, WT_REF *ref, WT_UPDATE *upd, ui { WT_CURSOR_BTREE cbt; WT_DECL_RET; + bool dryrun; + + dryrun = S2C(session)->rts->dryrun; __wt_btcur_init(session, &cbt); __wt_btcur_open(&cbt); @@ -183,11 +192,13 @@ __rts_btree_col_modify(WT_SESSION_IMPL *session, WT_REF *ref, WT_UPDATE *upd, ui WT_ERR(__wt_col_search(&cbt, recno, ref, true, NULL)); /* Apply the modification. */ + if (!dryrun) { #ifdef HAVE_DIAGNOSTIC - WT_ERR(__wt_col_modify(&cbt, recno, NULL, upd, WT_UPDATE_INVALID, true, false)); + WT_ERR(__wt_col_modify(&cbt, recno, NULL, upd, WT_UPDATE_INVALID, true, false)); #else - WT_ERR(__wt_col_modify(&cbt, recno, NULL, upd, WT_UPDATE_INVALID, true)); + WT_ERR(__wt_col_modify(&cbt, recno, NULL, upd, WT_UPDATE_INVALID, true)); #endif + } err: /* Free any resources that may have been cached in the cursor. */ @@ -205,6 +216,9 @@ __rts_btree_row_modify(WT_SESSION_IMPL *session, WT_REF *ref, WT_UPDATE *upd, WT { WT_CURSOR_BTREE cbt; WT_DECL_RET; + bool dryrun; + + dryrun = S2C(session)->rts->dryrun; __wt_btcur_init(session, &cbt); __wt_btcur_open(&cbt); @@ -213,11 +227,13 @@ __rts_btree_row_modify(WT_SESSION_IMPL *session, WT_REF *ref, WT_UPDATE *upd, WT WT_ERR(__wt_row_search(&cbt, key, true, ref, true, NULL)); /* Apply the modification. */ + if (!dryrun) { #ifdef HAVE_DIAGNOSTIC - WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true, false)); + WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true, false)); #else - WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true)); + WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true)); #endif + } err: /* Free any resources that may have been cached in the cursor. */ @@ -252,11 +268,14 @@ __rts_btree_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, uint8_t type; char ts_string[4][WT_TS_INT_STRING_SIZE]; char tw_string[WT_TIME_STRING_SIZE]; + bool dryrun; bool valid_update_found; #ifdef HAVE_DIAGNOSTIC bool first_record; #endif + dryrun = S2C(session)->rts->dryrun; + page = ref->page; hs_cursor = NULL; @@ -352,8 +371,10 @@ __rts_btree_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, "history store stop is obsolete with time window: %s and pinned timestamp: %s", __wt_time_window_to_string(hs_tw, tw_string), __wt_timestamp_to_string(pinned_ts, ts_string[0])); - WT_ERR(hs_cursor->remove(hs_cursor)); - WT_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); + if (!dryrun) + WT_ERR(hs_cursor->remove(hs_cursor)); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); + continue; } @@ -456,9 +477,10 @@ __rts_btree_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, first_record = false; #endif - WT_ERR(hs_cursor->remove(hs_cursor)); - WT_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); - WT_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts_unstable); + if (!dryrun) + WT_ERR(hs_cursor->remove(hs_cursor)); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); + WT_RTS_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts_unstable); } /* @@ -494,7 +516,7 @@ __rts_btree_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, * rollback to stable operation. */ F_SET(upd, WT_UPDATE_RESTORED_FROM_HS); - WT_STAT_CONN_DATA_INCR(session, txn_rts_hs_restore_updates); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_hs_restore_updates); /* * We have a tombstone on the original update chain and it is stable according to the @@ -537,11 +559,11 @@ __rts_btree_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, tombstone->next = upd; upd = tombstone; - WT_STAT_CONN_DATA_INCR(session, txn_rts_hs_restore_tombstones); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_hs_restore_tombstones); } } else { WT_ERR(__wt_upd_alloc_tombstone(session, &upd, NULL)); - WT_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); __wt_verbose_level_multi( session, WT_VERB_RECOVERY_RTS(session), WT_VERBOSE_DEBUG_3, "%s", "key removed"); } @@ -556,9 +578,10 @@ __rts_btree_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, /* Avoid freeing the updates while still in use if hs_cursor->remove fails. */ upd = tombstone = NULL; - WT_ERR(hs_cursor->remove(hs_cursor)); - WT_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); - WT_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts); + if (!dryrun) + WT_ERR(hs_cursor->remove(hs_cursor)); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); + WT_RTS_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts); } if (0) { @@ -574,6 +597,10 @@ err: __wt_scr_free(session, &key_string); if (hs_cursor != NULL) WT_TRET(hs_cursor->close(hs_cursor)); + if (dryrun) { + WT_ASSERT(session, !valid_update_found || upd == NULL); + __wt_free_update_list(session, &upd); + } return (ret); } @@ -621,7 +648,7 @@ __rts_btree_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, __wt_timestamp_to_string(vpack->tw.stop_ts, ts_string[3]), __wt_timestamp_to_string(rollback_timestamp, ts_string[4])); WT_RET(__wt_upd_alloc_tombstone(session, &upd, NULL)); - WT_STAT_CONN_DATA_INCR(session, txn_rts_sweep_hs_keys); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_sweep_hs_keys); } else return (0); } else if (vpack->tw.durable_start_ts > rollback_timestamp || @@ -646,7 +673,7 @@ __rts_btree_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, * rollback timestamp; thus, deleting it is the correct response. */ WT_RET(__wt_upd_alloc_tombstone(session, &upd, NULL)); - WT_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); } } else if (WT_TIME_WINDOW_HAS_STOP(&vpack->tw) && (vpack->tw.durable_stop_ts > rollback_timestamp || @@ -669,7 +696,7 @@ __rts_btree_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, * remove the key. */ WT_RET(__wt_upd_alloc_tombstone(session, &upd, NULL)); - WT_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_keys_removed); } } else { /* @@ -695,7 +722,7 @@ __rts_btree_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, upd->durable_ts = vpack->tw.durable_start_ts; upd->start_ts = vpack->tw.start_ts; F_SET(upd, WT_UPDATE_RESTORED_FROM_DS); - WT_STAT_CONN_DATA_INCR(session, txn_rts_keys_restored); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_keys_restored); __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), "key restored with commit timestamp: %s, durable timestamp: %s, stable timestamp: " "%s, " @@ -742,7 +769,7 @@ __rts_btree_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, else WT_ERR(__rts_btree_col_modify(session, ref, upd, recno)); - if (0) { + if (S2C(session)->rts->dryrun) { err: __wt_free(session, upd); } @@ -1039,7 +1066,9 @@ __wt_rts_btree_abort_updates( WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t rollback_timestamp) { WT_PAGE *page; - bool modified; + bool dryrun, modified; + + dryrun = S2C(session)->rts->dryrun; /* * If we have a ref with clean page, find out whether the page has any modifications that are @@ -1078,7 +1107,7 @@ __wt_rts_btree_abort_updates( } /* Mark the page as dirty to reconcile the page. */ - if (page->modify) + if (!dryrun && page->modify) __wt_page_modify_set(session, page); return (0); } diff --git a/src/third_party/wiredtiger/src/rollback_to_stable/rts_history.c b/src/third_party/wiredtiger/src/rollback_to_stable/rts_history.c index 2d1d4ecd292..42deb2f2836 100644 --- a/src/third_party/wiredtiger/src/rollback_to_stable/rts_history.c +++ b/src/third_party/wiredtiger/src/rollback_to_stable/rts_history.c @@ -20,6 +20,9 @@ __wt_rts_history_delete_hs(WT_SESSION_IMPL *session, WT_ITEM *key, wt_timestamp_ WT_DECL_ITEM(hs_key); WT_DECL_RET; WT_TIME_WINDOW *hs_tw; + bool dryrun; + + dryrun = S2C(session)->rts->dryrun; /* Open a history store table cursor. */ WT_RET(__wt_curhs_open(session, NULL, &hs_cursor)); @@ -51,17 +54,18 @@ __wt_rts_history_delete_hs(WT_SESSION_IMPL *session, WT_ITEM *key, wt_timestamp_ if (hs_tw->stop_ts <= ts) break; - WT_ERR(hs_cursor->remove(hs_cursor)); - WT_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); + if (!dryrun) + WT_ERR(hs_cursor->remove(hs_cursor)); + WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); /* - * The globally visible start time window's are cleared during history store reconciliation. + * The globally visible start time windows are cleared during history store reconciliation. * Treat them also as a stable entry removal from the history store. */ if (hs_tw->start_ts == ts || hs_tw->start_ts == WT_TS_NONE) - WT_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts); + WT_RTS_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts); else - WT_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts_unstable); + WT_RTS_STAT_CONN_DATA_INCR(session, cache_hs_key_truncate_rts_unstable); } WT_ERR_NOTFOUND_OK(ret, false); @@ -85,6 +89,9 @@ __wt_rts_history_btree_hs_truncate(WT_SESSION_IMPL *session, uint32_t btree_id) wt_timestamp_t hs_start_ts; uint64_t hs_counter; uint32_t hs_btree_id; + bool dryrun; + + dryrun = S2C(session)->rts->dryrun; hs_cursor_start = hs_cursor_stop = NULL; hs_btree_id = 0; @@ -129,10 +136,11 @@ __wt_rts_history_btree_hs_truncate(WT_SESSION_IMPL *session, uint32_t btree_id) hs_cursor_stop->get_key(hs_cursor_stop, &hs_btree_id, hs_key, &hs_start_ts, &hs_counter); } while (hs_btree_id != btree_id); - WT_ERR( - truncate_session->truncate(truncate_session, NULL, hs_cursor_start, hs_cursor_stop, NULL)); + if (!dryrun) + WT_ERR(truncate_session->truncate( + truncate_session, NULL, hs_cursor_start, hs_cursor_stop, NULL)); - WT_STAT_CONN_DATA_INCR(session, cache_hs_btree_truncate); + WT_RTS_STAT_CONN_DATA_INCR(session, cache_hs_btree_truncate); __wt_verbose(session, WT_VERB_RECOVERY_PROGRESS, "Rollback to stable has truncated records for btree %u from the history store", btree_id); diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index 65b84326e70..08b8623ce8e 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -203,7 +203,12 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(format_values, in_memory_values, prepare_values) + dryrun_values = [ + ('no_dryrun', dict(dryrun=False)), + ('dryrun', dict(dryrun=True)), + ] + + scenarios = make_scenarios(format_values, in_memory_values, prepare_values, dryrun_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -248,10 +253,13 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): if not self.in_memory: self.session.checkpoint() - self.conn.rollback_to_stable() + self.conn.rollback_to_stable("dryrun={}".format("true" if self.dryrun else "false")) # Check that the new updates are only seen after the update timestamp. self.session.breakpoint() - self.check(valuea, uri, nrows, None, 20) + if self.dryrun: + self.check(0, uri, nrows if self.value_format == '8t' else 0, None, 20) + else: + self.check(valuea, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] @@ -265,7 +273,9 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): self.assertEqual(calls, 1) self.assertEqual(hs_removed, 0) self.assertEqual(keys_removed, 0) - if self.in_memory: + if self.dryrun: + self.assertEqual(upd_aborted, 0) + elif self.in_memory: self.assertEqual(upd_aborted, nrows) else: self.assertEqual(upd_aborted + keys_restored, nrows) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py index c3d4dc39d45..1cd327e5373 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py @@ -62,7 +62,12 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(format_values, in_memory_values, prepare_values) + dryrun_values = [ + ('no_dryrun', dict(dryrun=False)), + ('dryrun', dict(dryrun=True)) + ] + + scenarios = make_scenarios(format_values, in_memory_values, prepare_values, dryrun_values) def conn_config(self): config = 'cache_size=100MB,statistics=(all)' @@ -122,10 +127,15 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): if not self.in_memory: self.session.checkpoint() - self.conn.rollback_to_stable() + self.conn.rollback_to_stable('dryrun={}'.format('true' if self.dryrun else 'false')) # Check that the new updates are only seen after the update timestamp. self.session.breakpoint() - self.check(valueb, uri, nrows, None, 40) + + if self.dryrun: + self.check(valued, uri, nrows, None, 40) + else: + self.check(valueb, uri, nrows, None, 40) + self.check(valueb, uri, nrows, None, 20) self.check(valuea, uri, nrows, None, 10) @@ -142,7 +152,11 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): self.assertEqual(keys_removed, 0) self.assertEqual(keys_restored, 0) self.assertGreater(pages_visited, 0) - self.assertGreaterEqual(upd_aborted, nrows * 2) + + if self.dryrun: + self.assertEqual(upd_aborted, 0) + else: + self.assertGreaterEqual(upd_aborted, nrows * 2) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py index 6bc9cda70a4..be75d500684 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py @@ -55,7 +55,12 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(format_values, in_memory_values, prepare_values) + dryrun_values = [ + ('no_dryrun', dict(dryrun=False)), + ('dryrun', dict(dryrun=True)) + ] + + scenarios = make_scenarios(format_values, in_memory_values, prepare_values, dryrun_values) def conn_config(self): config = 'cache_size=500MB,statistics=(all)' @@ -146,12 +151,16 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: self.session.checkpoint() - self.conn.rollback_to_stable() + self.conn.rollback_to_stable('dryrun={}'.format('true' if self.dryrun else 'false')) # Check that the correct data is seen at and after the stable timestamp. self.check(value_modQ, uri, nrows, None, 30) - self.check(value_modQ, uri, nrows, None, 150) - self.check(value_a, uri, nrows, None, 20) + if self.dryrun: + self.check(value_modZ, uri, nrows, None, 150) + self.check(value_a, uri, nrows, None, 20) + else: + self.check(value_modQ, uri, nrows, None, 150) + self.check(value_a, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] @@ -167,7 +176,9 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): self.assertEqual(keys_removed, 0) self.assertEqual(keys_restored, 0) self.assertGreater(pages_visited, 0) - if self.in_memory: + if self.dryrun: + self.assertEqual(upd_aborted + hs_removed + hs_sweep, 0) + elif self.in_memory: self.assertEqual(upd_aborted, nrows * 11) self.assertEqual(hs_removed + hs_sweep, 0) else: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py index ca51b63c7c2..be295341f56 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py @@ -48,7 +48,12 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(format_values, prepare_values) + dryrun_values = [ + ('no_dryrun', dict(dryrun=False)), + ('dryrun', dict(dryrun=True)), + ] + + scenarios = make_scenarios(format_values, prepare_values, dryrun_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -299,9 +304,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): self.check(None, uri, 0, nrows, 41 if self.prepare else 40) self.check(value_c, uri, nrows, None, 61 if self.prepare else 60) - self.conn.rollback_to_stable() + self.conn.rollback_to_stable("dryrun={}".format("true" if self.dryrun else "false")) # Perform several updates and checkpoint. - self.large_updates(uri, value_c, ds, nrows, self.prepare, 60) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 65 if self.dryrun else 60) self.session.checkpoint() # Simulate a server crash and restart. simulate_crash_restart(self, ".", "RESTART") @@ -311,4 +316,6 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): self.check(value_a, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2] + + # Unchanged due to shutdown/startup RTS. self.assertEqual(restored_tombstones, nrows) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable37.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable37.py index 8e856d791ec..f27d4ed35c2 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable37.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable37.py @@ -44,7 +44,12 @@ class test_rollback_to_stable37(test_rollback_to_stable_base): ('row_integer', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(format_values) + dryrun_values = [ + ('no_dryrun', dict(dryrun=False)), + ('dryrun', dict(dryrun=True)) + ] + + scenarios = make_scenarios(format_values, dryrun_values) def test_rollback_to_stable(self): uri = 'table:test_rollback_to_stable37' @@ -99,11 +104,15 @@ class test_rollback_to_stable37(test_rollback_to_stable_base): self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(2000)) self.session.checkpoint() - self.conn.rollback_to_stable() + self.conn.rollback_to_stable('dryrun={}'.format('true' if self.dryrun else 'false')) self.check(value_c, uri, nrows, None, 1000) self.check(value_c, uri, nrows, None, 2000) - self.check(value_c, uri, nrows, None, 3000) + + if self.dryrun: + self.check(value_d, uri, nrows, None, 3000) + else: + self.check(value_c, uri, nrows, None, 3000) stat_cursor = self.session.open_cursor('statistics:', None, None) keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable41.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable41.py new file mode 100644 index 00000000000..6c6abdf861a --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable41.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios +from test_rollback_to_stable01 import test_rollback_to_stable_base + +# test_rollback_to_stable41.py +# Test that the dry-run config for RTS only applies to a single call. +class test_rollback_to_stable41(test_rollback_to_stable_base): + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='S')), + ] + + scenarios = make_scenarios(format_values) + + def test_rollback_to_stable(self): + uri = 'table:test_rollback_to_stable41' + nrows = 1000 + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + else: + value_a = 'a' * 10 + value_b = 'b' * 10 + + # Create our table. + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format) + ds.populate() + + # Insert some data either side of the stable timestamp we set below. + self.large_updates(uri, value_a, ds, nrows, False, 10) + self.check(value_a, uri, nrows, None, 10) + self.large_updates(uri, value_b, ds, nrows, False, 30) + self.check(value_b, uri, nrows, None, 30) + + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) + + # Fake RTS, newer data should still exist. + self.conn.rollback_to_stable('dryrun=true') + self.check(value_b, uri, nrows, None, 30) + + # Real RTS, newer data should vanish. + self.conn.rollback_to_stable() + self.check(value_a, uri, nrows, None, 30) + +if __name__ == '__main__': + wttest.run() |