diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2020-05-29 16:48:08 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-05-29 07:01:36 +0000 |
| commit | d8292dbefc595e639164aa2e7d0317668781ca0f (patch) | |
| tree | 8134fa43e1b78ed47b5a6225d2c14fcf796a3d3f /src/third_party | |
| parent | 8f1df3e07ad034da2926d73130e0250c40e8e8ef (diff) | |
| download | mongo-d8292dbefc595e639164aa2e7d0317668781ca0f.tar.gz | |
Import wiredtiger: 943d9cdd2a1cfc4ae357da44ec497b5115ff3139 from branch mongodb-4.4
ref: a7af601643..943d9cdd2a
for: 4.4.0-rc8
WT-6168 Ignore history store visibility when resolving prepared updates
WT-6250 Avoid passing reconcile struct to cell layer for stat logging
WT-6302 Move format operations tracing into WiredTiger log files
WT-6303 Fix commit check may fail because of a prepared rollback
WT-6306 Do not force evict HS pages because of deleted items
WT-6307 Increase the page cache size for all the page allocations
WT-6312 Add history store verification
WT-6315 Fix commit check failure because of updates restored from data store
WT-6317 Create variant of compatibility test that runs faster and generates a smaller artifact footprint
WT-6318 Stop assuming exclusive access when writing to history store pages
WT-6320 Allow eviction of pages with older content
Diffstat (limited to 'src/third_party')
59 files changed, 1289 insertions, 904 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index b32f8455b21..84ea7bd9266 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -522,6 +522,14 @@ connection_runtime_config = [ is not limited to not skewing newest, not favoring leaf pages, and modifying the eviction score mechanism.''', type='boolean'), + Config('log_retention', '0', r''' + adjust log archiving to retain at least this number of log files, ignored if set to 0. + (Warning: this option can remove log files required for recovery if no checkpoints + have yet been done and the number of log files exceeds the configured value. As + WiredTiger cannot detect the difference between a system that has not yet checkpointed + and one that will never checkpoint, it might discard log files before any checkpoint is + done.)''', + min='0', max='1024'), Config('realloc_exact', 'false', r''' if true, reallocation of memory will only provide the exact amount requested. This will help with spotting memory allocation diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 843175370cc..914dc78087a 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -43,7 +43,8 @@ Barack BerkeleyDB Bitfield Bitwise -BlqRr +BlqR +BqRt Brueckner Bsearch Btree @@ -951,7 +952,9 @@ localkey localtime localvalue logf +logfile logmgr +logmsg lognum logop logpath @@ -963,7 +966,6 @@ lookaside lookup lookups lossy -lqRrt lr lrt lru @@ -1024,6 +1026,7 @@ multithreaded munmap mutex mutexes +mx mytable mytxn namespace @@ -1147,6 +1150,7 @@ pv pvA pwrite py +qRrT qdown qqq qrrSS diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 7040ca93354..f75d0bd1d4a 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -233,7 +233,6 @@ connection_stats = [ CacheStat('cache_eviction_fail_active_children_on_an_internal_page', 'pages selected for eviction unable to be evicted because of active children on an internal page'), CacheStat('cache_eviction_fail_in_reconciliation', 'pages selected for eviction unable to be evicted because of failure in reconciliation'), CacheStat('cache_eviction_fail_parent_has_overflow_items', 'pages selected for eviction unable to be evicted as the parent page has overflow items'), - CacheStat('cache_eviction_fail_with_newer_modifications_on_a_clean_page', 'pages selected for eviction unable to be evicted due to newer modifications on a clean page'), CacheStat('cache_eviction_force', 'forced eviction - pages selected count'), CacheStat('cache_eviction_force_clean', 'forced eviction - pages evicted that were clean count'), CacheStat('cache_eviction_force_clean_time', 'forced eviction - pages evicted that were clean time (usecs)'), @@ -546,6 +545,8 @@ connection_stats = [ RecStat('rec_time_aggr_oldest_start_ts', 'pages written including an aggregated oldest start timestamp '), RecStat('rec_time_aggr_oldest_start_txn', 'pages written including an aggregated oldest start transaction ID '), RecStat('rec_time_aggr_prepared', 'pages written including an aggregated prepare'), + RecStat('rec_time_window_bytes_ts', 'approximate byte size of timestamps in pages written'), + RecStat('rec_time_window_bytes_txn', 'approximate byte size of transaction IDs in pages written'), RecStat('rec_time_window_durable_start_ts', 'records written including a start durable timestamp'), RecStat('rec_time_window_durable_stop_ts', 'records written including a stop durable timestamp'), RecStat('rec_time_window_pages_durable_start_ts', 'pages written including at least one start durable timestamp'), @@ -874,6 +875,8 @@ dsrc_stats = [ RecStat('rec_time_aggr_oldest_start_ts', 'pages written including an aggregated oldest start timestamp '), RecStat('rec_time_aggr_oldest_start_txn', 'pages written including an aggregated oldest start transaction ID '), RecStat('rec_time_aggr_prepared', 'pages written including an aggregated prepare'), + RecStat('rec_time_window_bytes_ts', 'approximate byte size of timestamps in pages written'), + RecStat('rec_time_window_bytes_txn', 'approximate byte size of transaction IDs in pages written'), RecStat('rec_time_window_durable_start_ts', 'records written including a start durable timestamp'), RecStat('rec_time_window_durable_stop_ts', 'records written including a stop durable timestamp'), RecStat('rec_time_window_pages_durable_start_ts', 'pages written including at least one start durable timestamp'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 357159fff8d..dd5170decfd 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "a7af6016438884665a89bc7c67f42c7812472848" + "commit": "943d9cdd2a1cfc4ae357da44ec497b5115ff3139" } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index e3ad5538358..febfba92d77 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -688,10 +688,15 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) * and only saw deleted records, try to evict the page when we release it. Otherwise * repeatedly deleting from the beginning of a tree can have quadratic performance. Take * care not to force eviction of pages that are genuinely empty, in new trees. + * + * A visible stop timestamp could have been treated as a tombstone and accounted in the + * deleted count. Such a page might not have any new updates and be clean, but could benefit + * from reconciliation getting rid of the obsolete content. Hence mark the page dirty to + * force it through reconciliation. */ if (page != NULL && (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD || (newpage && cbt->page_deleted_count > 0))) { - __wt_page_evict_soon(session, cbt->ref); + WT_ERR(__wt_page_dirty_and_evict_soon(session, cbt->ref)); WT_STAT_CONN_INCR(session, cache_eviction_force_delete); } cbt->page_deleted_count = 0; diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index fa3718a6e87..2b934061ab3 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -646,10 +646,15 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) * and only saw deleted records, try to evict the page when we release it. Otherwise * repeatedly deleting from the beginning of a tree can have quadratic performance. Take * care not to force eviction of pages that are genuinely empty, in new trees. + * + * A visible stop timestamp could have been treated as a tombstone and accounted in the + * deleted count. Such a page might not have any new updates and be clean, but could benefit + * from reconciliation getting rid of the obsolete content. Hence mark the page dirty to + * force it through reconciliation. */ if (page != NULL && (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD || (newpage && cbt->page_deleted_count > 0))) { - __wt_page_evict_soon(session, cbt->ref); + WT_ERR(__wt_page_dirty_and_evict_soon(session, cbt->ref)); WT_STAT_CONN_INCR(session, cache_eviction_force_delete); } cbt->page_deleted_count = 0; diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index 60868afb3a5..7be09d8f514 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -294,7 +294,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) WT_ROW *rip; WT_TIME_WINDOW tw; WT_UPDATE **upd_array, *upd; - size_t size; + size_t size, total_size; uint32_t count, i; btree = S2BT(session); @@ -343,7 +343,8 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) * because deletes are instantiated after the history store table updates.) */ if (page->entries != 0 && page->modify->mod_row_update == NULL) - WT_RET(__wt_calloc_def(session, page->entries, &page->modify->mod_row_update)); + WT_PAGE_ALLOC_AND_SWAP( + session, page, page->modify->mod_row_update, upd_array, page->entries); /* * Allocate the per-reference update array; in the case of instantiating a page deleted in a @@ -362,15 +363,17 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) ++count; } WT_RET(__wt_calloc_def(session, count + 1, &page_del->update_list)); + __wt_cache_page_inmem_incr(session, page, (count + 1) * sizeof(page_del->update_list)); } /* Walk the page entries, giving each one a tombstone. */ - size = 0; + size = total_size = 0; count = 0; upd_array = page->modify->mod_row_update; if ((insert = WT_ROW_INSERT_SMALLEST(page)) != NULL) WT_SKIP_FOREACH (ins, insert) { WT_ERR(__tombstone_update_alloc(session, page_del, &upd, &size)); + total_size += size; upd->next = ins->upd; ins->upd = upd; @@ -385,6 +388,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) __wt_read_row_time_window(session, page, rip, &tw); if (!WT_TIME_WINDOW_HAS_STOP(&tw)) { WT_ERR(__tombstone_update_alloc(session, page_del, &upd, &size)); + total_size += size; upd->next = upd_array[WT_ROW_SLOT(page, rip)]; upd_array[WT_ROW_SLOT(page, rip)] = upd; @@ -394,6 +398,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) if ((insert = WT_ROW_INSERT(page, rip)) != NULL) WT_SKIP_FOREACH (ins, insert) { WT_ERR(__tombstone_update_alloc(session, page_del, &upd, &size)); + total_size += size; upd->next = ins->upd; ins->upd = upd; @@ -403,7 +408,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) } } - __wt_cache_page_inmem_incr(session, page, size); + __wt_cache_page_inmem_incr(session, page, total_size); return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index c6b2ebbe241..b3266462eda 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -535,7 +535,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) WT_DECL_ITEM(value); WT_DECL_RET; WT_ROW *rip; - WT_UPDATE *tombstone, **upd_array, *upd; + WT_UPDATE *tombstone, *upd, **upd_array; size_t size, total_size; uint32_t i; bool instantiate_prepared, prepare; @@ -599,7 +599,8 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) /* Allocate the per-page update array if one doesn't already exist. */ if (page->entries != 0 && page->modify->mod_row_update == NULL) - WT_RET(__wt_calloc_def(session, page->entries, &page->modify->mod_row_update)); + WT_PAGE_ALLOC_AND_SWAP( + session, page, page->modify->mod_row_update, upd_array, page->entries); /* For each entry in the page */ size = total_size = 0; @@ -614,6 +615,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value)); WT_ERR(__wt_upd_alloc(session, value, WT_UPDATE_STANDARD, &upd, &size)); + total_size += size; upd->durable_ts = unpack.tw.durable_start_ts; upd->start_ts = unpack.tw.start_ts; upd->txnid = unpack.tw.start_txn; @@ -625,22 +627,23 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) */ if (WT_TIME_WINDOW_HAS_STOP(&unpack.tw)) { WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size)); + total_size += size; tombstone->durable_ts = WT_TS_NONE; tombstone->start_ts = unpack.tw.stop_ts; tombstone->txnid = unpack.tw.stop_txn; tombstone->prepare_state = WT_PREPARE_INPROGRESS; - F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DISK); + F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS); + F_SET(upd, WT_UPDATE_RESTORED_FROM_DS); tombstone->next = upd; } else { upd->durable_ts = WT_TS_NONE; upd->prepare_state = WT_PREPARE_INPROGRESS; - F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DISK); + F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS); tombstone = upd; } upd_array[WT_ROW_SLOT(page, rip)] = tombstone; tombstone = upd = NULL; - total_size += size; } } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index d433fb02ff7..d5d99925797 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -55,6 +55,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure_compatibility_sub static const WT_CONFIG_CHECK confchk_wiredtiger_open_debug_mode_subconfigs[] = { {"checkpoint_retention", "int", NULL, "min=0,max=1024", NULL, 0}, {"cursor_copy", "boolean", NULL, NULL, NULL, 0}, {"eviction", "boolean", NULL, NULL, NULL, 0}, + {"log_retention", "int", NULL, "min=0,max=1024", NULL, 0}, {"realloc_exact", "boolean", NULL, NULL, NULL, 0}, {"rollback_error", "int", NULL, "min=0,max=10M", NULL, 0}, {"slow_checkpoint", "boolean", NULL, NULL, NULL, 0}, @@ -109,7 +110,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"compatibility", "category", NULL, NULL, confchk_WT_CONNECTION_reconfigure_compatibility_subconfigs, 1}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, {"error_prefix", "string", NULL, NULL, NULL, 0}, {"eviction", "category", NULL, NULL, confchk_wiredtiger_open_eviction_subconfigs, 2}, {"eviction_checkpoint_target", "int", NULL, "min=0,max=10TB", NULL, 0}, @@ -538,7 +539,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -611,7 +612,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, {"config_base", "boolean", NULL, NULL, NULL, 0}, {"create", "boolean", NULL, NULL, NULL, 0}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -683,7 +684,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -751,7 +752,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { {"checkpoint", "category", NULL, NULL, confchk_wiredtiger_open_checkpoint_subconfigs, 2}, {"checkpoint_sync", "boolean", NULL, NULL, NULL, 0}, {"compatibility", "category", NULL, NULL, confchk_wiredtiger_open_compatibility_subconfigs, 3}, - {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 7}, + {"debug_mode", "category", NULL, NULL, confchk_wiredtiger_open_debug_mode_subconfigs, 8}, {"direct_io", "list", NULL, "choices=[\"checkpoint\",\"data\",\"log\"]", NULL, 0}, {"encryption", "category", NULL, NULL, confchk_wiredtiger_open_encryption_subconfigs, 3}, {"error_prefix", "string", NULL, NULL, NULL, 0}, @@ -831,9 +832,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_overflow=(file_max=0),cache_overhead=8,cache_size=100MB," "checkpoint=(log_size=0,wait=0),compatibility=(release=)," "debug_mode=(checkpoint_retention=0,cursor_copy=false," - "eviction=false,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," + "eviction=false,log_retention=0,realloc_exact=false," + "rollback_error=0,slow_checkpoint=false,table_logging=false)," + "error_prefix=,eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1005,9 +1006,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," "debug_mode=(checkpoint_retention=0,cursor_copy=false," - "eviction=false,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=false,log_retention=0,realloc_exact=false," + "rollback_error=0,slow_checkpoint=false,table_logging=false)," + "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," "eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" @@ -1037,9 +1038,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),config_base=true,create=false," "debug_mode=(checkpoint_retention=0,cursor_copy=false," - "eviction=false,realloc_exact=false,rollback_error=0," - "slow_checkpoint=false,table_logging=false),direct_io=," - "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=false,log_retention=0,realloc_exact=false," + "rollback_error=0,slow_checkpoint=false,table_logging=false)," + "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," "eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" @@ -1069,11 +1070,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),debug_mode=(checkpoint_retention=0," - "cursor_copy=false,eviction=false,realloc_exact=false," - "rollback_error=0,slow_checkpoint=false,table_logging=false)," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "cursor_copy=false,eviction=false,log_retention=0," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_close_sync=true,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1100,11 +1101,11 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", ",cache_size=100MB,checkpoint=(log_size=0,wait=0)," "checkpoint_sync=true,compatibility=(release=,require_max=," "require_min=),debug_mode=(checkpoint_retention=0," - "cursor_copy=false,eviction=false,realloc_exact=false," - "rollback_error=0,slow_checkpoint=false,table_logging=false)," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=8,threads_min=1)," - "eviction_checkpoint_target=1,eviction_dirty_target=5," + "cursor_copy=false,eviction=false,log_retention=0," + "realloc_exact=false,rollback_error=0,slow_checkpoint=false," + "table_logging=false),direct_io=,encryption=(keyid=,name=," + "secretkey=),error_prefix=,eviction=(threads_max=8,threads_min=1)" + ",eviction_checkpoint_target=1,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",extensions=,file_close_sync=true,file_extend=," "file_manager=(close_handle_minimum=250,close_idle_time=30," diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index b07df9bb860..7ec08ed81c4 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1799,6 +1799,9 @@ __wt_debug_mode_config(WT_SESSION_IMPL *session, const char *cfg[]) else F_CLR(cache, WT_CACHE_EVICT_DEBUG_MODE); + WT_RET(__wt_config_gets(session, cfg, "debug_mode.log_retention", &cval)); + conn->debug_log_cnt = (uint32_t)cval.val; + WT_RET(__wt_config_gets(session, cfg, "debug_mode.realloc_exact", &cval)); if (cval.val) F_SET(conn, WT_CONN_DEBUG_REALLOC_EXACT); diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 1770505d566..09b5290b790 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -386,18 +386,28 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) * backup or the checkpoint LSN. Otherwise we want the minimum of the last log file written to * disk and the checkpoint LSN. */ - if (backup_file != 0) - min_lognum = WT_MIN(log->ckpt_lsn.l.file, backup_file); - else { + min_lognum = backup_file == 0 ? WT_MIN(log->ckpt_lsn.l.file, log->sync_lsn.l.file) : + WT_MIN(log->ckpt_lsn.l.file, backup_file); + + /* Adjust the number of log files to retain based on debugging options. */ + if (conn->debug_ckpt_cnt != 0) + min_lognum = WT_MIN(conn->debug_ckpt[conn->debug_ckpt_cnt - 1].l.file, min_lognum); + if (conn->debug_log_cnt != 0) { /* - * Figure out the minimum log file to archive. Use the LSN in the debugging array if - * necessary. + * If we're performing checkpoints, apply the retain value as a minimum, increasing the + * number the log files we keep. If not performing checkpoints, it's an absolute number of + * log files to keep. This means we can potentially remove log files required for recovery + * if the number of log files exceeds the configured value and the system has yet to be + * checkpointed. + * + * Check for N+1, that is, we retain N full log files, and one partial. */ - if (conn->debug_ckpt_cnt == 0) - min_lognum = WT_MIN(log->ckpt_lsn.l.file, log->sync_lsn.l.file); + if ((conn->debug_log_cnt + 1) >= log->fileid) + return (0); + if (log->ckpt_lsn.l.file == 1 && log->ckpt_lsn.l.offset == 0) + min_lognum = log->fileid - (conn->debug_log_cnt + 1); else - min_lognum = - WT_MIN(conn->debug_ckpt[conn->debug_ckpt_cnt - 1].l.file, log->sync_lsn.l.file); + min_lognum = WT_MIN(log->fileid - (conn->debug_log_cnt + 1), min_lognum); } __wt_verbose(session, WT_VERB_LOG, "log_archive: archive to log number %" PRIu32, min_lognum); diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index 53c36314ce9..e11d89567b4 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -325,7 +325,7 @@ Display the database log. The \c printlog command outputs the database log. @subsection util_printlog_synopsis Synopsis -<code>wt [-RrVv] [-C config] [-E secretkey ] [-h directory] printlog [-x] [-f output]</code> +<code>wt [-RrVv] [-C config] [-E secretkey ] [-h directory] printlog [-mx] [-f output]</code> @subsection util_printlog_options Options The following are command-specific options for the \c printlog command: @@ -334,6 +334,9 @@ The following are command-specific options for the \c printlog command: By default, the \c printlog command output is written to the standard output; the \c -f option re-directs the output to the specified file. +@par <code>-m</code> +Print only message-type log records. + @par <code>-x</code> Keys and value items in the log are printed in hex format in addition to the default string format. diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 2ffde8f8bcd..59e005fdb00 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -1915,20 +1915,6 @@ __evict_walk_tree(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_ent if (__wt_page_is_empty(page) || F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) goto fast; - /* - * If application threads are blocked on eviction of clean pages, and the only thing - * preventing a clean leaf page from being evicted is it contains historical data, mark it - * dirty so we can do history store eviction. We also mark the tree dirty to avoid an - * assertion that we don't discard dirty pages from a clean tree. - */ - if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN_HARD) && F_ISSET(ref, WT_REF_FLAG_LEAF) && - !modified && page->modify != NULL && - !__wt_txn_visible_all( - session, page->modify->rec_max_txn, page->modify->rec_max_timestamp)) { - __wt_page_modify_set(session, page); - goto fast; - } - /* Skip clean pages if appropriate. */ if (!modified && !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) continue; diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 5ecdf6bcc38..2c4f13578ed 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -336,15 +336,6 @@ __evict_page_clean_update(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) WT_DECL_RET; /* - * Before discarding a page, assert that all updates are globally visible unless the tree is - * closing or dead. - */ - WT_ASSERT(session, LF_ISSET(WT_EVICT_CALL_CLOSING) || ref->page->modify == NULL || - F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || - __wt_txn_visible_all(session, ref->page->modify->rec_max_txn, - ref->page->modify->rec_max_timestamp)); - - /* * Discard the page and update the reference structure. A page with a disk address is an on-disk * page, and a page without a disk address is a re-instantiated deleted page (for example, by * searching), that was never subsequently written. diff --git a/src/third_party/wiredtiger/src/history/hs.c b/src/third_party/wiredtiger/src/history/hs.c index 3f2f97cd248..b430e9bde5c 100644 --- a/src/third_party/wiredtiger/src/history/hs.c +++ b/src/third_party/wiredtiger/src/history/hs.c @@ -304,8 +304,12 @@ __wt_hs_modify(WT_CURSOR_BTREE *hs_cbt, WT_UPDATE *hs_upd) last_upd->next = mod->mod_row_update[hs_cbt->slot]; } + /* + * We don't have exclusive access to the history store page so we need to pass "false" here to + * ensure that we're locking when inserting new keys to an insert list. + */ WT_WITH_BTREE(session, CUR2BT(hs_cbt), - ret = __wt_row_modify(hs_cbt, &hs_cbt->iface.key, NULL, hs_upd, WT_UPDATE_INVALID, true)); + ret = __wt_row_modify(hs_cbt, &hs_cbt->iface.key, NULL, hs_upd, WT_UPDATE_INVALID, false)); return (ret); } @@ -1165,7 +1169,7 @@ __hs_delete_key_from_ts_int( hs_cursor = session->hs_cursor; WT_RET(__wt_scr_alloc(session, 0, &srch_key)); - hs_cursor->set_key(hs_cursor, btree_id, key, ts, (uint64_t)0); + hs_cursor->set_key(hs_cursor, btree_id, key, ts, 0); WT_ERR(__wt_buf_set(session, srch_key, hs_cursor->key.data, hs_cursor->key.size)); WT_ERR_NOTFOUND_OK(hs_cursor->search_near(hs_cursor, &exact), true); /* Empty history store is fine. */ @@ -1414,7 +1418,7 @@ __wt_history_store_verify_one(WT_SESSION_IMPL *session) * in the history store. */ memset(&hs_key, 0, sizeof(hs_key)); - cursor->set_key(cursor, btree_id, &hs_key, 0, 0, 0, 0); + cursor->set_key(cursor, btree_id, &hs_key, 0, 0); ret = cursor->search_near(cursor, &exact); if (ret == 0 && exact < 0) ret = cursor->next(cursor); diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index facf3ef247d..f74fc0a5cd6 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -171,6 +171,8 @@ #define SESSION_API_CALL_PREPARE_ALLOWED(s, n, config, cfg) \ API_CALL(s, WT_SESSION, n, NULL, config, cfg) +#define SESSION_API_CALL_PREPARE_ALLOWED_NOCONF(s, n) API_CALL_NOCONF(s, WT_SESSION, n, NULL) + #define SESSION_API_CALL_PREPARE_NOT_ALLOWED(s, n, config, cfg) \ SESSION_API_PREPARE_CHECK(s, WT_SESSION, n); \ API_CALL(s, WT_SESSION, n, NULL, config, cfg) diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index b5cf1a86ec2..c9711020d89 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -1073,10 +1073,11 @@ struct __wt_update { volatile uint8_t prepare_state; /* prepare state */ /* AUTOMATIC FLAG VALUE GENERATION START */ -#define WT_UPDATE_HS 0x1u /* Update has been written to history store. */ -#define WT_UPDATE_PREPARE_RESTORED_FROM_DISK 0x2u /* Prepared update restored from disk. */ -#define WT_UPDATE_RESTORED_FOR_ROLLBACK 0x4u /* Update restored for rollback to stable. */ - /* AUTOMATIC FLAG VALUE GENERATION STOP */ +#define WT_UPDATE_HS 0x1u /* Update has been written to history store. */ +#define WT_UPDATE_PREPARE_RESTORED_FROM_DS 0x2u /* Prepared update restored from data store. */ +#define WT_UPDATE_RESTORED_FROM_DS 0x4u /* Update restored from data store. */ +#define WT_UPDATE_RESTORED_FROM_HS 0x8u /* Update restored from history store. */ + /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint8_t flags; /* diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 3cbc1ed5e99..5a7fec77f71 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1410,14 +1410,6 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool *inmem_splitp) __wt_gen_active(session, WT_GEN_SPLIT, page->pg_intl_split_gen)) return (false); - /* - * If the page is clean but has modifications that appear too new to evict, skip it. - */ - if (!modified && !__wt_txn_visible_all(session, mod->rec_max_txn, mod->rec_max_timestamp)) { - WT_STAT_CONN_INCR(session, cache_eviction_fail_with_newer_modifications_on_a_clean_page); - return (false); - } - return (true); } diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i index d8d11943c94..3818e64dde7 100644 --- a/src/third_party/wiredtiger/src/include/cache.i +++ b/src/third_party/wiredtiger/src/include/cache.i @@ -101,6 +101,20 @@ __wt_page_evict_soon(WT_SESSION_IMPL *session, WT_REF *ref) } /* + * __wt_page_dirty_and_evict_soon -- + * Mark a page dirty and set it to be evicted as soon as possible. + */ +static inline int +__wt_page_dirty_and_evict_soon(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_RET(__wt_page_modify_init(session, ref->page)); + __wt_page_modify_set(session, ref->page); + __wt_page_evict_soon(session, ref); + + return (0); +} + +/* * __wt_cache_pages_inuse -- * Return the number of pages in use. */ diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index f105c6fb420..070e80e8718 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -32,8 +32,7 @@ __cell_check_value_validity(WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw, bool e * Pack the validity window for a value. */ static inline void -__cell_pack_value_validity( - WT_SESSION_IMPL *session, WT_RECONCILE *r, uint8_t **pp, WT_TIME_WINDOW *tw) +__cell_pack_value_validity(WT_SESSION_IMPL *session, uint8_t **pp, WT_TIME_WINDOW *tw) { uint8_t flags, *flagsp; @@ -85,15 +84,8 @@ __cell_pack_value_validity( LF_SET(WT_CELL_TS_DURABLE_STOP); } } - if (LF_ISSET( - WT_CELL_TS_START | WT_CELL_TS_DURABLE_START | WT_CELL_TS_STOP | WT_CELL_TS_DURABLE_STOP)) - r->rec_page_cell_with_ts = true; - if (LF_ISSET(WT_CELL_TXN_START | WT_CELL_TXN_STOP)) - r->rec_page_cell_with_txn_id = true; - if (tw->prepare) { + if (tw->prepare) LF_SET(WT_CELL_PREPARE); - r->rec_page_cell_with_prepared_txn = true; - } *flagsp = flags; } @@ -228,8 +220,8 @@ __wt_cell_pack_addr(WT_SESSION_IMPL *session, WT_CELL *cell, u_int cell_type, ui * Set a value item's WT_CELL contents. */ static inline size_t -__wt_cell_pack_value(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, - uint64_t rle, size_t size) +__wt_cell_pack_value( + WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, size_t size) { uint8_t byte, *p; bool validity; @@ -238,7 +230,7 @@ __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, W p = cell->__chunk; *p = '\0'; - __cell_pack_value_validity(session, r, &p, tw); + __cell_pack_value_validity(session, &p, tw); /* * Short data cells without a validity window or run-length encoding have 6 bits of data length @@ -360,8 +352,8 @@ __wt_cell_pack_value_match( * Write a copy value cell. */ static inline size_t -__wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, - uint64_t rle, uint64_t v) +__wt_cell_pack_copy( + WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, uint64_t v) { uint8_t *p; @@ -369,7 +361,7 @@ __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT p = cell->__chunk; *p = '\0'; - __cell_pack_value_validity(session, r, &p, tw); + __cell_pack_value_validity(session, &p, tw); if (rle < 2) cell->__chunk[0] |= WT_CELL_VALUE_COPY; /* Type */ @@ -389,8 +381,7 @@ __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT * Write a deleted value cell. */ static inline size_t -__wt_cell_pack_del( - WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle) +__wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle) { uint8_t *p; @@ -399,7 +390,7 @@ __wt_cell_pack_del( *p = '\0'; /* FIXME-WT-6124: we should set the time window prepare value. */ - __cell_pack_value_validity(session, r, &p, tw); + __cell_pack_value_validity(session, &p, tw); if (rle < 2) cell->__chunk[0] |= WT_CELL_DEL; /* Type */ @@ -485,8 +476,8 @@ __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size) * Pack an overflow cell. */ static inline size_t -__wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, uint8_t type, - WT_TIME_WINDOW *tw, uint64_t rle, size_t size) +__wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type, WT_TIME_WINDOW *tw, + uint64_t rle, size_t size) { uint8_t *p; @@ -502,7 +493,7 @@ __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, ui break; case WT_CELL_VALUE_OVFL: case WT_CELL_VALUE_OVFL_RM: - __cell_pack_value_validity(session, r, &p, tw); + __cell_pack_value_validity(session, &p, tw); break; } diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 4bf25bbb379..0295929c496 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -209,7 +209,8 @@ struct __wt_connection_impl { uintmax_t optrack_pid; /* Cache the process ID. */ WT_LSN *debug_ckpt; /* Debug mode checkpoint LSNs. */ - uint32_t debug_ckpt_cnt; /* Checkpoint retention number */ + uint32_t debug_ckpt_cnt; /* Checkpoint log file retention number */ + uint32_t debug_log_cnt; /* Log file retention count */ void **foc; /* Free-on-close array */ size_t foc_cnt; /* Array entries */ diff --git a/src/third_party/wiredtiger/src/include/cursor.i b/src/third_party/wiredtiger/src/include/cursor.i index d3237ad91d2..362efea63a9 100644 --- a/src/third_party/wiredtiger/src/include/cursor.i +++ b/src/third_party/wiredtiger/src/include/cursor.i @@ -210,9 +210,14 @@ __cursor_reset(WT_CURSOR_BTREE *cbt) /* * If we were scanning and saw a lot of deleted records on this page, try to evict the page when * we release it. + * + * A visible stop timestamp could have been treated as a tombstone and accounted in the deleted + * count. Such a page might not have any new updates and be clean, but could benefit from + * reconciliation getting rid of the obsolete content. Hence mark the page dirty to force it + * through reconciliation. */ if (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD) { - __wt_page_evict_soon(session, cbt->ref); + WT_RET(__wt_page_dirty_and_evict_soon(session, cbt->ref)); WT_STAT_CONN_INCR(session, cache_eviction_force_delete); } cbt->page_deleted_count = 0; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 5a125934d91..383ba2eb088 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -1947,6 +1947,8 @@ static inline int __wt_log_cmp(WT_LSN *lsn1, WT_LSN *lsn2) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_page_cell_data_ref(WT_SESSION_IMPL *session, WT_PAGE *page, void *unpack_arg, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_page_dirty_and_evict_soon(WT_SESSION_IMPL *session, WT_REF *ref) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_page_modify_init(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_page_parent_modify_set(WT_SESSION_IMPL *session, WT_REF *ref, bool page_only) @@ -2060,18 +2062,17 @@ static inline int __wt_write(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offse static inline size_t __wt_cell_pack_addr(WT_SESSION_IMPL *session, WT_CELL *cell, u_int cell_type, uint64_t recno, WT_TIME_AGGREGATE *ta, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline size_t __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, +static inline size_t __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, uint64_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline size_t __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, - WT_TIME_WINDOW *tw, uint64_t rle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline size_t __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, + uint64_t rle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline size_t __wt_cell_pack_int_key(WT_CELL *cell, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline size_t __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline size_t __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, - uint8_t type, WT_TIME_WINDOW *tw, uint64_t rle, size_t size) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline size_t __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, +static inline size_t __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type, + WT_TIME_WINDOW *tw, uint64_t rle, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline size_t __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline size_t __wt_cell_total_len(void *unpack_arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index c3c108a833e..bd99c32ca6e 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -397,6 +397,7 @@ struct __wt_txn_printlog_args { /* AUTOMATIC FLAG VALUE GENERATION START */ #define WT_TXN_PRINTLOG_HEX 0x1u /* Add hex output */ +#define WT_TXN_PRINTLOG_MSG 0x2u /* Messages only */ /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/reconcile.h b/src/third_party/wiredtiger/src/include/reconcile.h index 10f50a8c9b8..94a94a9dbc9 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.h +++ b/src/third_party/wiredtiger/src/include/reconcile.h @@ -256,8 +256,8 @@ struct __wt_reconcile { WT_CURSOR_BTREE update_modify_cbt; /* - * Variables to track reconciled pages containing cells with time window values and prepared - * transactions. + * Variables to track reconciliation calls for pages containing cells with time window values + * and prepared transactions. */ bool rec_page_cell_with_ts; bool rec_page_cell_with_txn_id; diff --git a/src/third_party/wiredtiger/src/include/reconcile.i b/src/third_party/wiredtiger/src/include/reconcile.i index 2bdb78d026f..449fac897de 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.i +++ b/src/third_party/wiredtiger/src/include/reconcile.i @@ -85,40 +85,70 @@ __rec_page_time_stats(WT_SESSION_IMPL *session, WT_RECONCILE *r) { /* Time window statistics */ if (r->count_durable_start_ts != 0) { + WT_STAT_CONN_INCRV( + session, rec_time_window_bytes_ts, r->count_durable_start_ts * sizeof(wt_timestamp_t)); + WT_STAT_DATA_INCRV( + session, rec_time_window_bytes_ts, r->count_durable_start_ts * sizeof(wt_timestamp_t)); WT_STAT_CONN_INCRV(session, rec_time_window_durable_start_ts, r->count_durable_start_ts); WT_STAT_DATA_INCRV(session, rec_time_window_durable_start_ts, r->count_durable_start_ts); WT_STAT_CONN_INCR(session, rec_time_window_pages_durable_start_ts); WT_STAT_DATA_INCR(session, rec_time_window_pages_durable_start_ts); + r->rec_page_cell_with_ts = true; } if (r->count_start_ts != 0) { + WT_STAT_CONN_INCRV( + session, rec_time_window_bytes_ts, r->count_start_ts * sizeof(wt_timestamp_t)); + WT_STAT_DATA_INCRV( + session, rec_time_window_bytes_ts, r->count_start_ts * sizeof(wt_timestamp_t)); WT_STAT_CONN_INCRV(session, rec_time_window_start_ts, r->count_start_ts); WT_STAT_DATA_INCRV(session, rec_time_window_start_ts, r->count_start_ts); WT_STAT_CONN_INCR(session, rec_time_window_pages_start_ts); WT_STAT_DATA_INCR(session, rec_time_window_pages_start_ts); + r->rec_page_cell_with_ts = true; } if (r->count_start_txn != 0) { + WT_STAT_CONN_INCRV( + session, rec_time_window_bytes_txn, r->count_start_txn * sizeof(uint64_t)); + WT_STAT_DATA_INCRV( + session, rec_time_window_bytes_txn, r->count_start_txn * sizeof(uint64_t)); WT_STAT_CONN_INCRV(session, rec_time_window_start_txn, r->count_start_txn); WT_STAT_DATA_INCRV(session, rec_time_window_start_txn, r->count_start_txn); WT_STAT_CONN_INCR(session, rec_time_window_pages_start_txn); WT_STAT_DATA_INCR(session, rec_time_window_pages_start_txn); + r->rec_page_cell_with_txn_id = true; } if (r->count_durable_stop_ts != 0) { + WT_STAT_CONN_INCRV( + session, rec_time_window_bytes_ts, r->count_durable_stop_ts * sizeof(wt_timestamp_t)); + WT_STAT_DATA_INCRV( + session, rec_time_window_bytes_ts, r->count_durable_stop_ts * sizeof(wt_timestamp_t)); WT_STAT_CONN_INCRV(session, rec_time_window_durable_stop_ts, r->count_durable_stop_ts); WT_STAT_DATA_INCRV(session, rec_time_window_durable_stop_ts, r->count_durable_stop_ts); WT_STAT_CONN_INCR(session, rec_time_window_pages_durable_stop_ts); WT_STAT_DATA_INCR(session, rec_time_window_pages_durable_stop_ts); + r->rec_page_cell_with_ts = true; } if (r->count_stop_ts != 0) { + WT_STAT_CONN_INCRV( + session, rec_time_window_bytes_ts, r->count_stop_ts * sizeof(wt_timestamp_t)); + WT_STAT_DATA_INCRV( + session, rec_time_window_bytes_ts, r->count_stop_ts * sizeof(wt_timestamp_t)); WT_STAT_CONN_INCRV(session, rec_time_window_stop_ts, r->count_stop_ts); WT_STAT_DATA_INCRV(session, rec_time_window_stop_ts, r->count_stop_ts); WT_STAT_CONN_INCR(session, rec_time_window_pages_stop_ts); WT_STAT_DATA_INCR(session, rec_time_window_pages_stop_ts); + r->rec_page_cell_with_ts = true; } if (r->count_stop_txn != 0) { + WT_STAT_CONN_INCRV( + session, rec_time_window_bytes_txn, r->count_stop_txn * sizeof(uint64_t)); + WT_STAT_DATA_INCRV( + session, rec_time_window_bytes_txn, r->count_stop_txn * sizeof(uint64_t)); WT_STAT_CONN_INCRV(session, rec_time_window_stop_txn, r->count_stop_txn); WT_STAT_DATA_INCRV(session, rec_time_window_stop_txn, r->count_stop_txn); WT_STAT_CONN_INCR(session, rec_time_window_pages_stop_txn); WT_STAT_DATA_INCR(session, rec_time_window_pages_stop_txn); + r->rec_page_cell_with_txn_id = true; } if (r->count_prepare != 0) { @@ -126,6 +156,7 @@ __rec_page_time_stats(WT_SESSION_IMPL *session, WT_RECONCILE *r) WT_STAT_DATA_INCRV(session, rec_time_window_prepared, r->count_prepare); WT_STAT_CONN_INCR(session, rec_time_window_pages_prepared); WT_STAT_DATA_INCR(session, rec_time_window_pages_prepared); + r->rec_page_cell_with_prepared_txn = true; } /* Time aggregate statistics */ @@ -352,7 +383,7 @@ __wt_rec_cell_build_val(WT_SESSION_IMPL *session, WT_RECONCILE *r, const void *d } __rec_cell_tw_stats(r, tw); - val->cell_len = __wt_cell_pack_value(session, r, &val->cell, tw, rle, val->buf.size); + val->cell_len = __wt_cell_pack_value(session, &val->cell, tw, rle, val->buf.size); val->len = val->cell_len + val->buf.size; return (0); @@ -400,7 +431,7 @@ __wt_rec_dict_replace( * offset from the beginning of the page. */ offset = (uint64_t)WT_PTRDIFF(r->first_free, (uint8_t *)r->cur_ptr->image.mem + dp->offset); - val->len = val->cell_len = __wt_cell_pack_copy(session, r, &val->cell, tw, rle, offset); + val->len = val->cell_len = __wt_cell_pack_copy(session, &val->cell, tw, rle, offset); val->buf.data = NULL; val->buf.size = 0; } diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index e832a0d4acf..907e0eac9af 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -441,7 +441,6 @@ struct __wt_connection_stats { int64_t cache_eviction_fail_parent_has_overflow_items; int64_t cache_eviction_fail_active_children_on_an_internal_page; int64_t cache_eviction_fail_in_reconciliation; - int64_t cache_eviction_fail_with_newer_modifications_on_a_clean_page; int64_t cache_eviction_walk; int64_t cache_write; int64_t cache_write_restore; @@ -615,6 +614,8 @@ struct __wt_connection_stats { int64_t perf_hist_opwrite_latency_lt1000; int64_t perf_hist_opwrite_latency_lt10000; int64_t perf_hist_opwrite_latency_gt10000; + int64_t rec_time_window_bytes_ts; + int64_t rec_time_window_bytes_txn; int64_t rec_page_delete_fast; int64_t rec_maximum_seconds; int64_t rec_pages; @@ -887,6 +888,8 @@ struct __wt_dsrc_stats { int64_t hs_gc_pages_evict; int64_t hs_gc_pages_removed; int64_t hs_gc_pages_visited; + int64_t rec_time_window_bytes_ts; + int64_t rec_time_window_bytes_txn; int64_t rec_dictionary; int64_t rec_page_delete_fast; int64_t rec_suffix_compression; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 0d00a0ce18b..9dc39f4d1b8 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -882,7 +882,7 @@ __wt_txn_read_upd_list( * rollback. */ if (prepare_updp != NULL && *prepare_updp == NULL && - F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DISK)) + F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS)) *prepare_updp = upd; continue; } @@ -948,9 +948,9 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint /* * If the stop time point is set, that means that there is a tombstone at that time. If it is - * not prepared and the it is visible to our txn then that means we've just spotted a tombstone - * and should return "not found", except for history store scan during rollback to stable and - * when we are told to ignore non-globally visible tombstones. + * not prepared and it is visible to our txn it means we've just spotted a tombstone and should + * return "not found", except scanning the history store during rollback to stable and when we + * are told to ignore non-globally visible tombstones. */ if (__wt_txn_tw_stop_visible(session, &tw) && ((!F_ISSET(&cbt->iface, WT_CURSTD_IGNORE_TOMBSTONE) && @@ -1008,7 +1008,7 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint * being updated to 30 and the reader not seeing it. */ if (prepare_upd != NULL) { - WT_ASSERT(session, F_ISSET(prepare_upd, WT_UPDATE_PREPARE_RESTORED_FROM_DISK)); + WT_ASSERT(session, F_ISSET(prepare_upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS)); if (prepare_upd->txnid == WT_TXN_ABORTED || prepare_upd->prepare_state == WT_PREPARE_RESOLVED) return (WT_RESTART); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 78ec3eab096..a3b2d5a79dd 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2192,19 +2192,27 @@ struct __wt_connection { * to change skew to force history store eviction to happen more aggressively. This * includes but is not limited to not skewing newest\, not favoring leaf pages\, and * modifying the eviction score mechanism., a boolean flag; default \c false.} - * @config{ realloc_exact, if true\, reallocation of memory will only - * provide the exact amount requested. This will help with spotting memory allocation - * issues more easily., a boolean flag; default \c false.} - * @config{ - * rollback_error, return a WT_ROLLBACK error from a transaction operation about every Nth - * operation to simulate a collision., an integer between 0 and 10M; default \c 0.} - * @config{ slow_checkpoint, if true\, slow down checkpoint creation - * by slowing down internal page processing., a boolean flag; default \c false.} - * @config{ table_logging, if true\, write transaction related - * information to the log for all operations\, even operations for tables with logging - * turned off. This setting introduces a log format change that may break older versions of - * WiredTiger. These operations are informational and skipped in recovery., a boolean flag; - * default \c false.} + * @config{ log_retention, adjust log archiving to retain at least + * this number of log files\, ignored if set to 0. (Warning: this option can remove log + * files required for recovery if no checkpoints have yet been done and the number of log + * files exceeds the configured value. As WiredTiger cannot detect the difference between a + * system that has not yet checkpointed and one that will never checkpoint\, it might + * discard log files before any checkpoint is done.)., an integer between 0 and 1024; + * default \c 0.} + * @config{ realloc_exact, if true\, reallocation of + * memory will only provide the exact amount requested. This will help with spotting memory + * allocation issues more easily., a boolean flag; default \c false.} + * @config{ rollback_error, return a WT_ROLLBACK error from a + * transaction operation about every Nth operation to simulate a collision., an integer + * between 0 and 10M; default \c 0.} + * @config{ slow_checkpoint, if + * true\, slow down checkpoint creation by slowing down internal page processing., a boolean + * flag; default \c false.} + * @config{ table_logging, if true\, write + * transaction related information to the log for all operations\, even operations for + * tables with logging turned off. This setting introduces a log format change that may + * break older versions of WiredTiger. These operations are informational and skipped in + * recovery., a boolean flag; default \c false.} * @config{ ),,} * @config{error_prefix, prefix string for error messages., a string; default empty.} * @config{eviction = (, eviction configuration options., a set of related configuration @@ -2809,6 +2817,12 @@ struct __wt_connection { * eviction, if true\, modify internal algorithms to change skew to force history store eviction to * happen more aggressively. This includes but is not limited to not skewing newest\, not favoring * leaf pages\, and modifying the eviction score mechanism., a boolean flag; default \c false.} + * @config{ log_retention, adjust log archiving to retain at least this + * number of log files\, ignored if set to 0. (Warning: this option can remove log files required + * for recovery if no checkpoints have yet been done and the number of log files exceeds the + * configured value. As WiredTiger cannot detect the difference between a system that has not yet + * checkpointed and one that will never checkpoint\, it might discard log files before any + * checkpoint is done.)., an integer between 0 and 1024; default \c 0.} * @config{ realloc_exact, if true\, reallocation of memory will only provide * the exact amount requested. This will help with spotting memory allocation issues more easily., * a boolean flag; default \c false.} @@ -5251,702 +5265,704 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * failure in reconciliation */ #define WT_STAT_CONN_CACHE_EVICTION_FAIL_IN_RECONCILIATION 1145 -/*! - * cache: pages selected for eviction unable to be evicted due to newer - * modifications on a clean page - */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL_WITH_NEWER_MODIFICATIONS_ON_A_CLEAN_PAGE 1146 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1147 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1146 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1148 +#define WT_STAT_CONN_CACHE_WRITE 1147 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1149 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1148 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1150 +#define WT_STAT_CONN_CACHE_OVERHEAD 1149 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1151 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1150 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1152 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1151 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1153 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1152 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1154 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1153 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1155 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1154 /*! capacity: background fsync file handles considered */ -#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1156 +#define WT_STAT_CONN_FSYNC_ALL_FH_TOTAL 1155 /*! capacity: background fsync file handles synced */ -#define WT_STAT_CONN_FSYNC_ALL_FH 1157 +#define WT_STAT_CONN_FSYNC_ALL_FH 1156 /*! capacity: background fsync time (msecs) */ -#define WT_STAT_CONN_FSYNC_ALL_TIME 1158 +#define WT_STAT_CONN_FSYNC_ALL_TIME 1157 /*! capacity: bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1159 +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1158 /*! capacity: bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1160 +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1159 /*! capacity: bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1161 +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1160 /*! capacity: bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1162 +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1161 /*! capacity: bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1163 +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1162 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1164 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1163 /*! capacity: time waiting due to total capacity (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1165 +#define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1164 /*! capacity: time waiting during checkpoint (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1166 +#define WT_STAT_CONN_CAPACITY_TIME_CKPT 1165 /*! capacity: time waiting during eviction (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1167 +#define WT_STAT_CONN_CAPACITY_TIME_EVICT 1166 /*! capacity: time waiting during logging (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_LOG 1168 +#define WT_STAT_CONN_CAPACITY_TIME_LOG 1167 /*! capacity: time waiting during read (usecs) */ -#define WT_STAT_CONN_CAPACITY_TIME_READ 1169 +#define WT_STAT_CONN_CAPACITY_TIME_READ 1168 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1170 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1169 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1171 +#define WT_STAT_CONN_COND_AUTO_WAIT 1170 /*! connection: detected system time went backwards */ -#define WT_STAT_CONN_TIME_TRAVEL 1172 +#define WT_STAT_CONN_TIME_TRAVEL 1171 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1173 +#define WT_STAT_CONN_FILE_OPEN 1172 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1174 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1173 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1175 +#define WT_STAT_CONN_MEMORY_FREE 1174 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1176 +#define WT_STAT_CONN_MEMORY_GROW 1175 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1177 +#define WT_STAT_CONN_COND_WAIT 1176 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1178 +#define WT_STAT_CONN_RWLOCK_READ 1177 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1179 +#define WT_STAT_CONN_RWLOCK_WRITE 1178 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1180 +#define WT_STAT_CONN_FSYNC_IO 1179 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1181 +#define WT_STAT_CONN_READ_IO 1180 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1182 +#define WT_STAT_CONN_WRITE_IO 1181 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1183 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1182 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1184 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1183 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1185 +#define WT_STAT_CONN_CURSOR_CACHE 1184 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1186 +#define WT_STAT_CONN_CURSOR_CREATE 1185 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1187 +#define WT_STAT_CONN_CURSOR_INSERT 1186 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1188 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1187 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1189 +#define WT_STAT_CONN_CURSOR_MODIFY 1188 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1190 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1189 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1191 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1190 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1192 +#define WT_STAT_CONN_CURSOR_NEXT 1191 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1193 +#define WT_STAT_CONN_CURSOR_RESTART 1192 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1194 +#define WT_STAT_CONN_CURSOR_PREV 1193 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1195 +#define WT_STAT_CONN_CURSOR_REMOVE 1194 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1196 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1195 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1197 +#define WT_STAT_CONN_CURSOR_RESERVE 1196 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1198 +#define WT_STAT_CONN_CURSOR_RESET 1197 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1199 +#define WT_STAT_CONN_CURSOR_SEARCH 1198 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1200 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1199 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1201 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1200 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1202 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1201 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1203 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1202 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1204 +#define WT_STAT_CONN_CURSOR_SWEEP 1203 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1205 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1204 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1206 +#define WT_STAT_CONN_CURSOR_UPDATE 1205 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1207 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1206 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1208 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1207 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1209 +#define WT_STAT_CONN_CURSOR_REOPEN 1208 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1210 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1209 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1211 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1210 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1212 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1211 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1213 +#define WT_STAT_CONN_DH_SWEEP_REF 1212 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1214 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1213 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1215 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1214 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1216 +#define WT_STAT_CONN_DH_SWEEP_TOD 1215 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1217 +#define WT_STAT_CONN_DH_SWEEPS 1216 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1218 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1217 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1219 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1218 /*! history: history pages added for eviction during garbage collection */ -#define WT_STAT_CONN_HS_GC_PAGES_EVICT 1220 +#define WT_STAT_CONN_HS_GC_PAGES_EVICT 1219 /*! history: history pages removed for garbage collection */ -#define WT_STAT_CONN_HS_GC_PAGES_REMOVED 1221 +#define WT_STAT_CONN_HS_GC_PAGES_REMOVED 1220 /*! history: history pages visited for garbage collection */ -#define WT_STAT_CONN_HS_GC_PAGES_VISITED 1222 +#define WT_STAT_CONN_HS_GC_PAGES_VISITED 1221 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1223 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1222 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1224 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1223 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1225 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1224 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1226 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1225 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1227 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1226 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1228 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1227 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1229 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1228 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1230 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1229 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1231 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1230 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1232 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1231 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1233 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1232 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1234 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1233 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1235 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1234 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1236 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1235 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1237 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1236 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1238 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1237 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1239 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1238 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1240 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1239 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1241 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1240 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1242 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1241 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1243 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1242 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1244 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1243 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1245 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1244 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1246 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1245 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1247 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1246 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1248 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1247 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1249 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1248 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1250 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1249 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1251 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1250 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1252 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1251 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1253 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1252 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1254 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1253 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1255 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1254 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1256 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1255 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1257 +#define WT_STAT_CONN_LOG_FLUSH 1256 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1258 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1257 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1259 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1258 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1260 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1259 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1261 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1260 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1262 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1261 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1263 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1262 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1264 +#define WT_STAT_CONN_LOG_SCANS 1263 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1265 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1264 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1266 +#define WT_STAT_CONN_LOG_WRITE_LSN 1265 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1267 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1266 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1268 +#define WT_STAT_CONN_LOG_SYNC 1267 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1269 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1268 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1270 +#define WT_STAT_CONN_LOG_SYNC_DIR 1269 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1271 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1270 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1272 +#define WT_STAT_CONN_LOG_WRITES 1271 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1273 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1272 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1274 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1273 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1275 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1274 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1276 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1275 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1277 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1276 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1278 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1277 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1279 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1278 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1280 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1279 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1281 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1280 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1282 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1281 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1283 +#define WT_STAT_CONN_LOG_SLOT_RACES 1282 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1284 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1283 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1285 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1284 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1286 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1285 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1287 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1286 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1288 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1287 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1289 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1288 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1290 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1289 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1291 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1290 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1292 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1291 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1293 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1292 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1294 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1293 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1295 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1294 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1296 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1295 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1297 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1296 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1298 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1297 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1299 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1298 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1300 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1299 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1301 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1300 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1302 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1301 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1303 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1302 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1304 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1303 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1305 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1304 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1306 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1305 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1307 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1306 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1308 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1307 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1309 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1308 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1310 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1309 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1311 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1310 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1312 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1311 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1313 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1312 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1314 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1313 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1315 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1314 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1316 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1315 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1317 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1316 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1318 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1317 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1319 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1318 +/*! reconciliation: approximate byte size of timestamps in pages written */ +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1319 +/*! + * reconciliation: approximate byte size of transaction IDs in pages + * written + */ +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1320 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1320 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1321 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1321 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1322 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1322 +#define WT_STAT_CONN_REC_PAGES 1323 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1323 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1324 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1324 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1325 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1325 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1326 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1326 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1327 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1327 +#define WT_STAT_CONN_REC_PAGE_DELETE 1328 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1328 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1329 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1329 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1330 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1330 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1331 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1331 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1332 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1332 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1333 /*! * reconciliation: pages written including an aggregated oldest start * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TXN 1333 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TXN 1334 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1334 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1335 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1335 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1336 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1336 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1337 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1337 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1338 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1338 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1339 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1339 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1340 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1340 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1341 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1341 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1342 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1342 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1343 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1343 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1344 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1344 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1345 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1345 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1346 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1346 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1347 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1347 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1348 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1348 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1349 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1349 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1350 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1350 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1351 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1351 +#define WT_STAT_CONN_SESSION_OPEN 1352 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1352 +#define WT_STAT_CONN_SESSION_QUERY_TS 1353 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1353 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1354 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1354 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1355 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1355 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1356 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1356 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1357 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1357 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1358 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1358 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1359 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1359 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1360 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1360 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1361 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1361 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1362 /*! session: table import failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1362 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1363 /*! session: table import successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1363 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1364 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1364 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1365 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1365 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1366 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1366 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1367 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1367 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1368 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1368 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1369 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1369 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1370 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1370 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1371 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1371 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1372 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1372 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1373 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1373 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1374 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1374 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1375 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1375 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1376 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1376 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1377 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1377 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1378 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1378 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1379 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1379 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1380 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1380 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1381 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1381 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1382 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1382 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1383 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1383 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1384 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1384 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1385 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1385 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1386 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1386 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1387 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1387 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1388 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1388 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1389 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1389 +#define WT_STAT_CONN_PAGE_SLEEP 1390 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1390 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1391 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1391 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1392 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1392 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1393 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1393 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1394 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1394 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1395 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1395 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1396 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1396 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1397 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1397 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1398 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1398 +#define WT_STAT_CONN_TXN_PREPARE 1399 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1399 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1400 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1400 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1401 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1401 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1402 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1402 +#define WT_STAT_CONN_TXN_QUERY_TS 1403 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1403 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1404 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1404 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1405 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1405 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1406 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1406 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1407 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1407 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1408 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1408 +#define WT_STAT_CONN_TXN_RTS 1409 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1409 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1410 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1410 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1411 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1411 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1412 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1412 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1413 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1413 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1414 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1414 +#define WT_STAT_CONN_TXN_SET_TS 1415 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1415 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1416 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1416 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1417 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1417 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1418 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1418 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1419 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1419 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1420 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1420 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1421 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1421 +#define WT_STAT_CONN_TXN_BEGIN 1422 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1422 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1423 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1423 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1424 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1424 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1425 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1425 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1426 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1426 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1427 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1427 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1428 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1428 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1429 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1429 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1430 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1430 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1431 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1431 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1432 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1432 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1433 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1433 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1434 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1434 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1435 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1435 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1436 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1436 +#define WT_STAT_CONN_TXN_CHECKPOINT 1437 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1437 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1438 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1438 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1439 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1439 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1440 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1440 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1441 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1441 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1442 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1442 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1443 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1443 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1444 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1444 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1445 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1445 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1446 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1446 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1447 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1447 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1448 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1448 +#define WT_STAT_CONN_TXN_SYNC 1449 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1449 +#define WT_STAT_CONN_TXN_COMMIT 1450 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1450 +#define WT_STAT_CONN_TXN_ROLLBACK 1451 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1451 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1452 /*! * @} @@ -6332,113 +6348,120 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_HS_GC_PAGES_REMOVED 2131 /*! history: history pages visited for garbage collection */ #define WT_STAT_DSRC_HS_GC_PAGES_VISITED 2132 +/*! reconciliation: approximate byte size of timestamps in pages written */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TS 2133 +/*! + * reconciliation: approximate byte size of transaction IDs in pages + * written + */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TXN 2134 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2133 +#define WT_STAT_DSRC_REC_DICTIONARY 2135 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2134 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2136 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2135 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2137 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2136 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2138 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2137 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2139 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2138 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2140 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2139 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2141 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2140 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2142 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2141 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2143 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2142 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2144 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2143 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2145 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2144 +#define WT_STAT_DSRC_REC_PAGES 2146 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2145 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2147 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2146 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2148 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2147 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2149 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2148 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2150 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2149 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2151 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2150 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2152 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2151 +#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2153 /*! * reconciliation: pages written including an aggregated oldest start * transaction ID */ -#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TXN 2152 +#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TXN 2154 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2153 +#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2155 /*! reconciliation: pages written including at least one prepare */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_PREPARED 2154 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_PREPARED 2156 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2155 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2157 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TS 2156 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TS 2158 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2157 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2159 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2158 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2160 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2159 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2161 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2160 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2162 /*! reconciliation: records written including a prepare */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PREPARED 2161 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PREPARED 2163 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2162 +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2164 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2163 +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2165 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2164 +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2166 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2165 +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2167 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2166 +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2168 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2167 +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2169 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2168 +#define WT_STAT_DSRC_SESSION_COMPACT 2170 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2169 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2171 /*! * @} diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index bb6bd93884f..dc4487321dc 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -118,7 +118,7 @@ __wt_bulk_insert_var(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool delet val = &r->v; if (deleted) { - val->cell_len = __wt_cell_pack_del(session, r, &val->cell, &tw, cbulk->rle); + val->cell_len = __wt_cell_pack_del(session, &val->cell, &tw, cbulk->rle); val->buf.data = NULL; val->buf.size = 0; val->len = val->cell_len; @@ -526,13 +526,13 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_SALVAGE_COOKI } if (deleted) { - val->cell_len = __wt_cell_pack_del(session, r, &val->cell, tw, rle); + val->cell_len = __wt_cell_pack_del(session, &val->cell, tw, rle); val->buf.data = NULL; val->buf.size = 0; val->len = val->cell_len; } else if (overflow_type) { val->cell_len = - __wt_cell_pack_ovfl(session, r, &val->cell, WT_CELL_VALUE_OVFL, tw, rle, value->size); + __wt_cell_pack_ovfl(session, &val->cell, WT_CELL_VALUE_OVFL, tw, rle, value->size); val->buf.data = value->data; val->buf.size = value->size; val->len = val->cell_len + value->size; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 9ee0a8987e7..0bf79f2802a 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -802,8 +802,8 @@ __wt_rec_row_leaf( val->buf.size = vpack->size; /* Rebuild the cell. */ - val->cell_len = __wt_cell_pack_ovfl( - session, r, &val->cell, vpack->raw, &tw, 0, val->buf.size); + val->cell_len = + __wt_cell_pack_ovfl(session, &val->cell, vpack->raw, &tw, 0, val->buf.size); val->len = val->cell_len + val->buf.size; } else WT_ERR(__rec_cell_repack(session, btree, r, vpack, &tw)); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index 758a47c00d4..b4a054a95b1 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -69,11 +69,8 @@ __rec_append_orig_value( /* Review the current update list, checking conditions that mean no work is needed. */ for (;; upd = upd->next) { - /* - * Done if the update was restored from the history store for the rollback to stable - * operation. - */ - if (F_ISSET(upd, WT_UPDATE_RESTORED_FOR_ROLLBACK)) + /* Done if the update was restored from the history store. */ + if (F_ISSET(upd, WT_UPDATE_RESTORED_FROM_HS)) return (0); /* @@ -155,6 +152,7 @@ __rec_append_orig_value( tombstone->txnid = unpack->tw.stop_txn; tombstone->start_ts = unpack->tw.stop_ts; tombstone->durable_ts = unpack->tw.durable_stop_ts; + F_SET(tombstone, WT_UPDATE_RESTORED_FROM_DS); } else { /* * Once the prepared update is resolved, the in-memory update and on-disk written copy @@ -182,6 +180,7 @@ __rec_append_orig_value( append->txnid = unpack->tw.start_txn; append->start_ts = unpack->tw.start_ts; append->durable_ts = unpack->tw.durable_start_ts; + F_SET(append, WT_UPDATE_RESTORED_FROM_DS); } if (tombstone != NULL) { @@ -337,9 +336,9 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v * the history store, rollback to stable will bring it back to the data store. * Otherwise, it removes the key. */ - WT_ASSERT(session, F_ISSET(r, WT_REC_EVICT) || - (F_ISSET(r, WT_REC_VISIBILITY_ERR) && - F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DISK))); + WT_ASSERT(session, + F_ISSET(r, WT_REC_EVICT) || (F_ISSET(r, WT_REC_VISIBILITY_ERR) && + F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS))); WT_ASSERT(session, upd->prepare_state == WT_PREPARE_INPROGRESS); } } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index d083f076c69..7452fea82ca 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -2337,7 +2337,7 @@ __wt_rec_cell_build_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_KV *k WT_ERR(__wt_buf_set(session, &kv->buf, addr, size)); /* Build the cell and return. */ - kv->cell_len = __wt_cell_pack_ovfl(session, r, &kv->cell, type, tw, rle, kv->buf.size); + kv->cell_len = __wt_cell_pack_ovfl(session, &kv->cell, type, tw, rle, kv->buf.size); kv->len = kv->cell_len + kv->buf.size; err: diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 459a0757032..cceccec0ee8 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -813,7 +813,7 @@ __session_log_printf(WT_SESSION *wt_session, const char *fmt, ...) va_list ap; session = (WT_SESSION_IMPL *)wt_session; - SESSION_API_CALL_PREPARE_NOT_ALLOWED_NOCONF(session, log_printf); + SESSION_API_CALL_PREPARE_ALLOWED_NOCONF(session, log_printf); va_start(ap, fmt); ret = __wt_log_vprintf(session, fmt, ap); diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 4470dfccfa0..704c26c40dc 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -83,8 +83,10 @@ static const char *const __stats_dsrc_desc[] = { "cursor: update calls", "cursor: update key and value bytes", "cursor: update value size change", "history: history pages added for eviction during garbage collection", "history: history pages removed for garbage collection", - "history: history pages visited for garbage collection", "reconciliation: dictionary matches", - "reconciliation: fast-path pages deleted", + "history: history pages visited for garbage collection", + "reconciliation: approximate byte size of timestamps in pages written", + "reconciliation: approximate byte size of transaction IDs in pages written", + "reconciliation: dictionary matches", "reconciliation: fast-path pages deleted", "reconciliation: internal page key bytes discarded using suffix compression", "reconciliation: internal page multi-block writes", "reconciliation: internal-page overflow keys", "reconciliation: leaf page key bytes discarded using prefix compression", @@ -287,6 +289,8 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->hs_gc_pages_evict = 0; stats->hs_gc_pages_removed = 0; stats->hs_gc_pages_visited = 0; + stats->rec_time_window_bytes_ts = 0; + stats->rec_time_window_bytes_txn = 0; stats->rec_dictionary = 0; stats->rec_page_delete_fast = 0; stats->rec_suffix_compression = 0; @@ -483,6 +487,8 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->hs_gc_pages_evict += from->hs_gc_pages_evict; to->hs_gc_pages_removed += from->hs_gc_pages_removed; to->hs_gc_pages_visited += from->hs_gc_pages_visited; + to->rec_time_window_bytes_ts += from->rec_time_window_bytes_ts; + to->rec_time_window_bytes_txn += from->rec_time_window_bytes_txn; to->rec_dictionary += from->rec_dictionary; to->rec_page_delete_fast += from->rec_page_delete_fast; to->rec_suffix_compression += from->rec_suffix_compression; @@ -677,6 +683,8 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->hs_gc_pages_evict += WT_STAT_READ(from, hs_gc_pages_evict); to->hs_gc_pages_removed += WT_STAT_READ(from, hs_gc_pages_removed); to->hs_gc_pages_visited += WT_STAT_READ(from, hs_gc_pages_visited); + to->rec_time_window_bytes_ts += WT_STAT_READ(from, rec_time_window_bytes_ts); + to->rec_time_window_bytes_txn += WT_STAT_READ(from, rec_time_window_bytes_txn); to->rec_dictionary += WT_STAT_READ(from, rec_dictionary); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_suffix_compression += WT_STAT_READ(from, rec_suffix_compression); @@ -829,8 +837,6 @@ static const char *const __stats_connection_desc[] = { "cache: pages selected for eviction unable to be evicted because of active children on an " "internal page", "cache: pages selected for eviction unable to be evicted because of failure in reconciliation", - "cache: pages selected for eviction unable to be evicted due to newer modifications on a clean " - "page", "cache: pages walked for eviction", "cache: pages written from cache", "cache: pages written requiring in-memory restoration", "cache: percentage overhead", "cache: tracked bytes belonging to internal pages in the cache", @@ -942,6 +948,8 @@ static const char *const __stats_connection_desc[] = { "perf: operation write latency histogram (bucket 3) - 500-999us", "perf: operation write latency histogram (bucket 4) - 1000-9999us", "perf: operation write latency histogram (bucket 5) - 10000us+", + "reconciliation: approximate byte size of timestamps in pages written", + "reconciliation: approximate byte size of transaction IDs in pages written", "reconciliation: fast-path pages deleted", "reconciliation: maximum seconds spent in a reconciliation call", "reconciliation: page reconciliation calls", @@ -1234,7 +1242,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_fail_parent_has_overflow_items = 0; stats->cache_eviction_fail_active_children_on_an_internal_page = 0; stats->cache_eviction_fail_in_reconciliation = 0; - stats->cache_eviction_fail_with_newer_modifications_on_a_clean_page = 0; stats->cache_eviction_walk = 0; stats->cache_write = 0; stats->cache_write_restore = 0; @@ -1408,6 +1415,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->perf_hist_opwrite_latency_lt1000 = 0; stats->perf_hist_opwrite_latency_lt10000 = 0; stats->perf_hist_opwrite_latency_gt10000 = 0; + stats->rec_time_window_bytes_ts = 0; + stats->rec_time_window_bytes_txn = 0; stats->rec_page_delete_fast = 0; /* not clearing rec_maximum_seconds */ stats->rec_pages = 0; @@ -1721,8 +1730,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * WT_STAT_READ(from, cache_eviction_fail_active_children_on_an_internal_page); to->cache_eviction_fail_in_reconciliation += WT_STAT_READ(from, cache_eviction_fail_in_reconciliation); - to->cache_eviction_fail_with_newer_modifications_on_a_clean_page += - WT_STAT_READ(from, cache_eviction_fail_with_newer_modifications_on_a_clean_page); to->cache_eviction_walk += WT_STAT_READ(from, cache_eviction_walk); to->cache_write += WT_STAT_READ(from, cache_write); to->cache_write_restore += WT_STAT_READ(from, cache_write_restore); @@ -1900,6 +1907,8 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->perf_hist_opwrite_latency_lt1000 += WT_STAT_READ(from, perf_hist_opwrite_latency_lt1000); to->perf_hist_opwrite_latency_lt10000 += WT_STAT_READ(from, perf_hist_opwrite_latency_lt10000); to->perf_hist_opwrite_latency_gt10000 += WT_STAT_READ(from, perf_hist_opwrite_latency_gt10000); + to->rec_time_window_bytes_ts += WT_STAT_READ(from, rec_time_window_bytes_ts); + to->rec_time_window_bytes_txn += WT_STAT_READ(from, rec_time_window_bytes_txn); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_maximum_seconds += WT_STAT_READ(from, rec_maximum_seconds); to->rec_pages += WT_STAT_READ(from, rec_pages); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index af9aac63b11..fd19865636f 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -704,7 +704,7 @@ __txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_ITEM * * Set the flag to indicate that this update has been restored from history store for the * rollback of a prepared transaction. */ - F_SET(upd, WT_UPDATE_RESTORED_FOR_ROLLBACK); + F_SET(upd, WT_UPDATE_RESTORED_FROM_HS); /* Walk to the end of the chain and we can only have prepared updates on the update chain. */ for (;; chain = chain->next) { @@ -909,7 +909,7 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * * For prepared delete, we don't need to fix the history store. */ - if (F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DISK) && upd->type != WT_UPDATE_TOMBSTONE) { + if (F_ISSET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS) && upd->type != WT_UPDATE_TOMBSTONE) { cbt = (WT_CURSOR_BTREE *)(*cursorp); hs_btree_id = S2BT(session)->id; /* Open a history store table cursor. */ @@ -922,9 +922,12 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * Scan the history store for the given btree and key with maximum start timestamp to let * the search point to the last version of the key. We must ignore tombstone in the history * store while retrieving the update from the history store to replace the update in the - * data store. + * data store. We also need to ignore visibility of the updates as we have already released + * our snapshot in prepare. Otherwise, we can't see updates with non-globally visible + * transaction ids. */ F_SET(hs_cursor, WT_CURSTD_IGNORE_TOMBSTONE); + F_SET(session, WT_SESSION_HS_IGNORE_VISIBILITY); WT_ERR_NOTFOUND_OK( __wt_hs_cursor_position(session, hs_cursor, hs_btree_id, &op->u.op_row.key, WT_TS_MAX), true); @@ -1009,6 +1012,7 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, err: if (hs_cursor != NULL) { + F_CLR(session, WT_SESSION_HS_IGNORE_VISIBILITY); F_CLR(hs_cursor, WT_CURSTD_IGNORE_TOMBSTONE); ret = __wt_hs_cursor_close(session, session_flags, is_owner); } @@ -1035,7 +1039,6 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) txn = session->txn; cursor = NULL; - durable_op_timestamp = prev_op_timestamp = WT_TS_NONE; /* * Debugging checks on timestamps, if user requested them. @@ -1095,25 +1098,32 @@ __txn_commit_timestamps_assert(WT_SESSION_IMPL *session) (upd->txnid == WT_TXN_ABORTED || upd->txnid == WT_TXN_NONE || upd->txnid == txn->id)) upd = upd->next; + if (upd == NULL) + continue; + /* * Check the timestamp on this update with the first valid update in the chain. They're in * most recent order. */ - if (upd != NULL) { - prev_op_timestamp = upd->start_ts; - durable_op_timestamp = upd->durable_ts; - } + prev_op_timestamp = upd->start_ts; + durable_op_timestamp = upd->durable_ts; - if (upd == NULL) - continue; /* * Check for consistent per-key timestamp usage. If timestamps are or are not used * originally then they should be used the same way always. For this transaction, timestamps * are in use anytime the commit timestamp is set. Check timestamps are used in order. + * + * We may see an update restored from the data store or the history store with 0 timestamp + * if that update is behind the oldest timestamp when the page is reconciled. If the update + * is restored from the history store, it is either appended by the prepared rollback or + * rollback to stable. If the update is restored from the data store, it is either + * instantiated along with the prepared stop when the page is read into memory or appended + * by a failed eviction which attempted to write a prepared update to the data store. */ op_zero_ts = !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT); upd_zero_ts = prev_op_timestamp == WT_TS_NONE; - if (op_zero_ts != upd_zero_ts) { + if (op_zero_ts != upd_zero_ts && + !F_ISSET(upd, WT_UPDATE_RESTORED_FROM_HS | WT_UPDATE_RESTORED_FROM_DS)) { WT_ERR(__wt_verbose_dump_update(session, upd)); WT_ERR(__wt_verbose_dump_txn_one(session, session, EINVAL, "per-key timestamps used inconsistently, dumping relevant information")); diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index e52cc139906..d541d8f48d5 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -660,6 +660,17 @@ __txn_printlog(WT_SESSION_IMPL *session, WT_ITEM *rawrec, WT_LSN *lsnp, WT_LSN * /* First, peek at the log record type. */ WT_RET(__wt_logrec_read(session, &p, end, &rectype)); + /* + * When printing just the message records, display the message by itself without the usual log + * header information. + */ + if (F_ISSET(args, WT_TXN_PRINTLOG_MSG)) { + if (rectype != WT_LOGREC_MESSAGE) + return (0); + WT_RET(__wt_struct_unpack(session, p, WT_PTRDIFF(end, p), WT_UNCHECKED_STRING(S), &msg)); + return (__wt_fprintf(session, args->fs, "%s\n", msg)); + } + if (!firstrecord) WT_RET(__wt_fprintf(session, args->fs, ",\n")); @@ -734,11 +745,13 @@ __wt_txn_printlog(WT_SESSION *wt_session, const char *ofile, uint32_t flags) WT_RET( __wt_fopen(session, ofile, WT_FS_OPEN_CREATE | WT_FS_OPEN_FIXED, WT_STREAM_WRITE, &fs)); - WT_ERR(__wt_fprintf(session, fs, "[\n")); + if (!LF_ISSET(WT_TXN_PRINTLOG_MSG)) + WT_ERR(__wt_fprintf(session, fs, "[\n")); args.fs = fs; args.flags = flags; WT_ERR(__wt_log_scan(session, NULL, WT_LOGSCAN_FIRST, __txn_printlog, &args)); - ret = __wt_fprintf(session, fs, "\n]\n"); + if (!LF_ISSET(WT_TXN_PRINTLOG_MSG)) + ret = __wt_fprintf(session, fs, "\n]\n"); err: if (ofile != NULL) diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index bd22db20023..bb14f58d593 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -312,7 +312,7 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW * Set the flag to indicate that this update has been restored from history store for * the rollback to stable operation. */ - F_SET(upd, WT_UPDATE_RESTORED_FOR_ROLLBACK); + F_SET(upd, WT_UPDATE_RESTORED_FROM_HS); } else { WT_ERR(__wt_upd_alloc_tombstone(session, &upd, NULL)); WT_STAT_CONN_INCR(session, txn_rts_keys_removed); diff --git a/src/third_party/wiredtiger/src/utilities/util_printlog.c b/src/third_party/wiredtiger/src/utilities/util_printlog.c index ee46e98159f..9d09470bbd9 100644 --- a/src/third_party/wiredtiger/src/utilities/util_printlog.c +++ b/src/third_party/wiredtiger/src/utilities/util_printlog.c @@ -28,11 +28,14 @@ util_printlog(WT_SESSION *session, int argc, char *argv[]) flags = 0; ofile = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "f:x")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "f:mx")) != EOF) switch (ch) { case 'f': /* output file */ ofile = __wt_optarg; break; + case 'm': /* messages only */ + LF_SET(WT_TXN_PRINTLOG_MSG); + break; case 'x': /* hex output */ LF_SET(WT_TXN_PRINTLOG_HEX); break; diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 2f9877a7083..c2e748513e2 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1514,6 +1514,17 @@ tasks: ulimit -c unlimited largescale/run-million-collection-test.sh . + - name: compatibility-test-for-releases-full + commands: + - func: "get project" + - command: shell.exec + params: + working_dir: "wiredtiger" + script: | + set -o errexit + set -o verbose + test/evergreen/compatibility_test_for_releases.sh -l + - name: compatibility-test-for-releases commands: - func: "get project" @@ -2345,9 +2356,16 @@ buildvariants: tasks: - name: million-collection-test +- name: compatibility-tests-full + display_name: Compatibility tests full + batchtime: 10080 # 7 days + run_on: + - ubuntu1804-test + tasks: + - name: compatibility-test-for-releases-full + - name: compatibility-tests display_name: Compatibility tests - batchtime: 10080 # 7 days run_on: - ubuntu1804-test tasks: diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh index 5dfd5554689..5a396aad3dc 100755 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash ############################################################################################## # Check branches to ensure forward/backward compatibility, including some upgrade/downgrade testing. +# Pass in -l to trigger the longer version of this test which tests more variants. ############################################################################################## set -e @@ -149,14 +150,43 @@ upgrade_downgrade() done } +############################################################# +# usage strinf +############################################################# +usage() +{ + echo "Usage: \tcompatibility_test_for_releases [-l]" + echo "\t-l\trun additional variants of wiredtiger" + exit 1 +} + +long=false + +if [ $# -eq 1 ]; then + if [ $1 = "-l" ]; then + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Performing long compatibility test run" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + long=true + else + (usage) + fi +else + if [ $# -gt 1 ]; then + (usage) + fi +fi + # Create a directory in which to do the work. top="test-compatibility-run" rm -rf "$top" && mkdir "$top" cd "$top" # Build the branches. -(build_branch mongodb-3.4) -(build_branch mongodb-3.6) +if [ "$long" = true ]; then + (build_branch mongodb-3.4) + (build_branch mongodb-3.6) +fi (build_branch mongodb-4.0) (build_branch mongodb-4.2) (build_branch mongodb-4.4) @@ -165,30 +195,40 @@ cd "$top" # Get the names of the last two WiredTiger releases, wt1 is the most recent release, wt2 is the # release before that. Minor trickiness, we depend on the "develop" directory already existing # so we have a source in which to do git commands. -cd develop; wt1=$(get_prev_version 1); cd .. -(build_branch "$wt1") -cd develop; wt2=$(get_prev_version 2); cd .. -(build_branch "$wt2") +if [ "$long" = true ]; then + cd develop; wt1=$(get_prev_version 1); cd .. + (build_branch "$wt1") + cd develop; wt2=$(get_prev_version 2); cd .. + (build_branch "$wt2") +fi # Run format in each branch for supported access methods. -(run_format mongodb-3.4 "fix row var") -(run_format mongodb-3.6 "fix row var") +if [ "$long" = true ]; then + (run_format mongodb-3.4 "fix row var") + (run_format mongodb-3.6 "fix row var") +fi (run_format mongodb-4.0 "fix row var") (run_format mongodb-4.2 "fix row var") (run_format mongodb-4.4 "row") (run_format develop "row") -(run_format "$wt1" "fix row var") -(run_format "$wt2" "fix row var") +if [ "$long" = true ]; then + (run_format "$wt1" "fix row var") + (run_format "$wt2" "fix row var") +fi # Verify backward compatibility for supported access methods. -(verify_branches mongodb-3.6 mongodb-3.4 "fix row var") -(verify_branches mongodb-4.0 mongodb-3.6 "fix row var") +if [ "$long" = true ]; then + (verify_branches mongodb-3.6 mongodb-3.4 "fix row var") + (verify_branches mongodb-4.0 mongodb-3.6 "fix row var") +fi (verify_branches mongodb-4.2 mongodb-4.0 "fix row var") (verify_branches mongodb-4.4 mongodb-4.2 "fix row var") (verify_branches develop mongodb-4.4 "row") (verify_branches develop mongodb-4.2 "row") -(verify_branches "$wt1" "$wt2" "row") -(verify_branches develop "$wt1" "row") +if [ "$long" = true ]; then + (verify_branches "$wt1" "$wt2" "row") + (verify_branches develop "$wt1" "row") +fi # Verify forward compatibility for supported access methods. (verify_branches mongodb-4.2 mongodb-4.4 "row") diff --git a/src/third_party/wiredtiger/test/format/Makefile.am b/src/third_party/wiredtiger/test/format/Makefile.am index 034876933f7..59dd1c8a4d4 100644 --- a/src/third_party/wiredtiger/test/format/Makefile.am +++ b/src/third_party/wiredtiger/test/format/Makefile.am @@ -4,8 +4,8 @@ AM_CPPFLAGS +=-I$(top_srcdir)/test/utility noinst_PROGRAMS = t t_SOURCES =\ - backup.c bulk.c checkpoint.c compact.c config.c config_compat.c kv.c ops.c random.c \ - rebalance.c salvage.c snap.c t.c util.c wts.c + backup.c bulk.c checkpoint.c compact.c config.c config_compat.c hs.c kv.c ops.c random.c \ + rebalance.c salvage.c snap.c t.c trace.c util.c wts.c t_LDADD = $(top_builddir)/test/utility/libtest_util.la t_LDADD +=$(top_builddir)/libwiredtiger.la diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index b16835c7eec..ca87d27f098 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -45,9 +45,7 @@ check_copy(void) path = dmalloc(len); testutil_check(__wt_snprintf(path, len, "%s/BACKUP", g.home)); - wts_open(path, false, &conn, true); - - testutil_checkfmt(conn->open_session(conn, NULL, NULL, &session), "%s", path); + wts_open(path, &conn, &session, true); /* * Verify can return EBUSY if the handle isn't available. Don't yield and retry, in the case of @@ -56,7 +54,7 @@ check_copy(void) ret = session->verify(session, g.uri, NULL); testutil_assertfmt(ret == 0 || ret == EBUSY, "WT_SESSION.verify: %s: %s", path, g.uri); - testutil_checkfmt(conn->close(conn, NULL), "%s", path); + wts_close(&conn, &session); free(path); } diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index 69f986aa79a..e0694c3c058 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -87,7 +87,7 @@ wts_load(void) testutil_check(conn->open_session(conn, NULL, NULL, &session)); - logop(session, "%s", "=============== bulk load start"); + tracemsg("%s", "=============== bulk load start"); /* * No bulk load with custom collators, the order of insertion will not match the collation @@ -133,21 +133,24 @@ wts_load(void) if (!is_bulk) cursor->set_key(cursor, keyno); cursor->set_value(cursor, *(uint8_t *)value.data); - logop(session, "%-10s %" PRIu32 " {0x%02" PRIx8 "}", "bulk", keyno, - ((uint8_t *)value.data)[0]); + if (g.trace_all) + tracemsg( + "%-10s %" PRIu32 " {0x%02" PRIx8 "}", "bulk", keyno, ((uint8_t *)value.data)[0]); break; case VAR: if (!is_bulk) cursor->set_key(cursor, keyno); cursor->set_value(cursor, &value); - logop(session, "%-10s %" PRIu32 " {%.*s}", "bulk", keyno, (int)value.size, - (char *)value.data); + if (g.trace_all) + tracemsg( + "%-10s %" PRIu32 " {%.*s}", "bulk", keyno, (int)value.size, (char *)value.data); break; case ROW: cursor->set_key(cursor, &key); cursor->set_value(cursor, &value); - logop(session, "%-10s %" PRIu32 " {%.*s}, {%.*s}", "bulk", keyno, (int)key.size, - (char *)key.data, (int)value.size, (char *)value.data); + if (g.trace_all) + tracemsg("%-10s %" PRIu32 " {%.*s}, {%.*s}", "bulk", keyno, (int)key.size, + (char *)key.data, (int)value.size, (char *)value.data); break; } @@ -196,7 +199,7 @@ wts_load(void) testutil_check(cursor->close(cursor)); - logop(session, "%s", "=============== bulk load stop"); + tracemsg("%s", "=============== bulk load stop"); testutil_check(session->close(session, NULL)); diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 6bfeb640c06..b48bc0e18a2 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -29,7 +29,6 @@ #include "format.h" #include "config.h" -static void config(void); static void config_backup_incr(void); static void config_backward_compatible(void); static void config_cache(void); @@ -68,8 +67,6 @@ static void config_transaction(void); void config_final(void) { - config(); /* Finish up configuration and review it. */ - config_print(false); g.rows = g.c_rows; /* Set the key count. */ @@ -82,8 +79,8 @@ config_final(void) * config -- * Initialize the configuration itself. */ -static void -config(void) +void +config_run(void) { CONFIG *cp; char buf[128]; @@ -390,7 +387,11 @@ config_cache(void) * This code is what dramatically increases the cache size when there are lots of threads, it * grows the cache to several megabytes per thread. */ - workers = g.c_threads + (g.c_random_cursor ? 1 : 0); + workers = g.c_threads; + if (g.c_hs_cursor) + ++workers; + if (g.c_random_cursor) + ++workers; g.c_cache = WT_MAX(g.c_cache, 2 * workers * g.c_memory_page_max); /* @@ -968,8 +969,8 @@ config_print(bool error_display) CONFIG *cp; FILE *fp; - /* Reopening or replaying an existing database should leave the existing CONFIG file. */ - if (g.reopen || g.replay) + /* Reopening an existing database should leave the existing CONFIG file. */ + if (g.reopen) return; if (error_display) diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 04da641386b..3ad4571d41f 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -227,6 +227,10 @@ static CONFIG c[] = { /* 10% */ {"ops.compaction", "if compaction is running", C_BOOL, 10, 0, 0, &g.c_compact, NULL}, + /* 50% */ + {"ops.history_store", "if history store cursor reads configured", C_BOOL, 50, 0, 0, + &g.c_hs_cursor, NULL}, + {"ops.pct.delete", "percent operations that are deletes", C_IGNORE, 0, 0, 100, &g.c_delete_pct, NULL}, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 9f864aeec32..93e3ab54003 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -54,7 +54,6 @@ #define WT_NAME "wt" /* Object name */ #define DATASOURCE(v) (strcmp(v, g.c_data_source) == 0 ? 1 : 0) -#define SINGLETHREADED (g.c_threads == 1) #define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ @@ -75,14 +74,16 @@ typedef struct { #define LOCK_INITIALIZED(lock) ((lock)->lock_type != LOCK_NONE) typedef struct { + wt_thread_t tid; /* thread ID */ + char tidbuf[128]; /* thread ID in printable form */ + WT_CONNECTION *wts_conn; - WT_EXTENSION_API *wt_api; + WT_CONNECTION *wts_conn_inmemory; char *uri; /* Object name */ bool backward_compatible; /* Backward compatibility testing */ bool reopen; /* Reopen an existing database */ - bool replay; /* Replaying a run. */ bool workers_finished; /* Operations completed */ char *home; /* Home directory */ @@ -95,16 +96,15 @@ typedef struct { char *home_rand; /* RNG log file path */ char *home_stats; /* Statistics file path */ - char *config_open; /* Command-line configuration */ - char wiredtiger_open_config[8 * 1024]; /* Database open config */ - - bool rand_log_stop; /* Logging turned off */ - FILE *randfp; /* Random number log */ + char *config_open; /* Command-line configuration */ uint32_t run_cnt; /* Run counter */ - bool logging; /* log operations */ - FILE *logfp; /* log file */ + bool trace; /* trace operations */ + bool trace_all; /* trace all operations */ + bool trace_local; /* write trace to the primary database */ + WT_CONNECTION *trace_conn; /* optional tracing database */ + WT_SESSION *trace_session; RWLOCK backup_lock; /* Backup running */ uint64_t backup_id; /* Block incremental id */ @@ -135,8 +135,8 @@ typedef struct { uint32_t c_assert_commit_timestamp; uint32_t c_assert_read_timestamp; uint32_t c_auto_throttle; - uint32_t c_backups; char *c_backup_incremental; + uint32_t c_backups; uint32_t c_bitcnt; uint32_t c_bloom; uint32_t c_bloom_bit_count; @@ -161,6 +161,7 @@ typedef struct { uint32_t c_evict_max; char *c_file_type; uint32_t c_firstfit; + uint32_t c_hs_cursor; uint32_t c_huffman_key; uint32_t c_huffman_value; uint32_t c_in_memory; @@ -299,8 +300,9 @@ typedef struct { } SNAP_OPS; typedef struct { - int id; /* simple thread ID */ - wt_thread_t tid; /* thread ID */ + int id; /* simple thread ID */ + wt_thread_t tid; /* thread ID */ + char tidbuf[128]; /* thread ID in printable form */ WT_RAND_STATE rnd; /* thread RNG state */ @@ -319,6 +321,8 @@ typedef struct { WT_SESSION *session; /* WiredTiger session */ WT_CURSOR *cursor; /* WiredTiger cursor */ + WT_SESSION *trace; /* WiredTiger operations tracing session */ + uint64_t keyno; /* key */ WT_ITEM *key, _key; /* key, value */ WT_ITEM *value, _value; @@ -345,27 +349,24 @@ typedef struct { } TINFO; extern TINFO **tinfo_list; -#define logop(wt_session, fmt, ...) \ - do { \ - if (g.logging) \ - testutil_check(g.wt_api->msg_printf(g.wt_api, wt_session, fmt, __VA_ARGS__)); \ - } while (0) - WT_THREAD_RET alter(void *); WT_THREAD_RET backup(void *); WT_THREAD_RET checkpoint(void *); WT_THREAD_RET compact(void *); +WT_THREAD_RET hs_cursor(void *); +WT_THREAD_RET random_kv(void *); +WT_THREAD_RET timestamp(void *); + void config_clear(void); void config_compat(const char **); void config_error(void); void config_file(const char *); void config_final(void); void config_print(bool); +void config_run(void); void config_single(const char *, bool); void fclose_and_clear(FILE **); -bool fp_readv(FILE *, char *, bool, uint32_t *); -void handle_init(void); -void handle_teardown(void); +bool fp_readv(FILE *, char *, uint32_t *); void key_gen_common(WT_ITEM *, uint64_t, const char *); void key_gen_init(WT_ITEM *); void key_gen_teardown(WT_ITEM *); @@ -373,10 +374,7 @@ void key_init(void); void lock_destroy(WT_SESSION *, RWLOCK *); void lock_init(WT_SESSION *, RWLOCK *); void operations(u_int, bool); -WT_THREAD_RET random_kv(void *); void path_setup(const char *); -int read_row_worker(WT_CURSOR *, uint64_t, WT_ITEM *, WT_ITEM *, bool); -uint32_t rng_slow(WT_RAND_STATE *); void set_alarm(u_int); void set_core_off(void); void snap_init(TINFO *, uint64_t, bool); @@ -384,20 +382,22 @@ void snap_repeat_single(WT_CURSOR *, TINFO *); int snap_repeat_txn(WT_CURSOR *, TINFO *); void snap_repeat_update(TINFO *, bool); void snap_track(TINFO *, thread_op); -WT_THREAD_RET timestamp(void *); void timestamp_once(WT_SESSION *); +int trace_config(const char *); +void trace_init(void); +void trace_ops_init(TINFO *); +void trace_teardown(void); void track(const char *, uint64_t, TINFO *); void val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t); void val_gen_init(WT_ITEM *); void val_gen_teardown(WT_ITEM *); void val_init(void); void wts_checkpoints(void); -void wts_close(void); -void wts_create(void); +void wts_close(WT_CONNECTION **, WT_SESSION **); +void wts_create(const char *); void wts_dump(const char *, bool); -void wts_init(void); void wts_load(void); -void wts_open(const char *, bool, WT_CONNECTION **, bool); +void wts_open(const char *, WT_CONNECTION **, WT_SESSION **, bool); void wts_read_scan(void); void wts_rebalance(void); void wts_reopen(void); diff --git a/src/third_party/wiredtiger/test/format/format.i b/src/third_party/wiredtiger/test/format/format.i index e1634ff13d3..43055bb2809 100644 --- a/src/third_party/wiredtiger/test/format/format.i +++ b/src/third_party/wiredtiger/test/format/format.i @@ -97,12 +97,7 @@ rng(WT_RAND_STATE *rnd) if (rnd == NULL) rnd = &g.rnd; - /* - * Multithreaded runs log/replay until they get to the operations phase, then turn off logging - * and replay because threaded operation order can't be replayed. Do that check inline so it's a - * cheap call once thread performance starts to matter. - */ - return (g.randfp == NULL || g.rand_log_stop ? __wt_random(rnd) : rng_slow(rnd)); + return (__wt_random(rnd)); } /* @@ -236,3 +231,25 @@ lock_writeunlock(WT_SESSION *session, RWLOCK *lock) testutil_check(pthread_rwlock_unlock(&lock->l.pthread)); } } +#define tracemsg(fmt, ...) \ + do { \ + if (g.trace) { \ + struct timespec __ts; \ + WT_SESSION *__s = g.trace_session; \ + __wt_epoch((WT_SESSION_IMPL *)__s, &__ts); \ + testutil_check( \ + __s->log_printf(__s, "[%" PRIuMAX ":%" PRIuMAX "][%s] " fmt, (uintmax_t)__ts.tv_sec, \ + (uintmax_t)__ts.tv_nsec / WT_THOUSAND, g.tidbuf, __VA_ARGS__)); \ + } \ + } while (0) +#define traceop(tinfo, fmt, ...) \ + do { \ + if (g.trace) { \ + struct timespec __ts; \ + WT_SESSION *__s = (tinfo)->trace; \ + __wt_epoch((WT_SESSION_IMPL *)__s, &__ts); \ + testutil_check( \ + __s->log_printf(__s, "[%" PRIuMAX ":%" PRIuMAX "][%s] " fmt, (uintmax_t)__ts.tv_sec, \ + (uintmax_t)__ts.tv_nsec / WT_THOUSAND, tinfo->tidbuf, __VA_ARGS__)); \ + } \ + } while (0) diff --git a/src/third_party/wiredtiger/test/format/hs.c b/src/third_party/wiredtiger/test/format/hs.c new file mode 100644 index 00000000000..e5d40e6256b --- /dev/null +++ b/src/third_party/wiredtiger/test/format/hs.c @@ -0,0 +1,124 @@ +/*- + * Public Domain 2014-2020 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "format.h" + +/* + * hs_cursor -- + * Do history store cursor operations. + */ +WT_THREAD_RET +hs_cursor(void *arg) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_ITEM hs_key, hs_value; + WT_SESSION *session; + wt_timestamp_t hs_durable_timestamp, hs_start_ts, hs_stop_durable_ts; + uint64_t hs_counter, hs_upd_type; + uint32_t hs_btree_id, i; + u_int period; + int exact; + bool restart; + + (void)(arg); /* Unused parameter */ + + conn = g.wts_conn; + + /* + * Trigger the internal WiredTiger cursor order checking on the history-store file. Open a + * cursor on the history-store file, retrieve some records, close cursor, repeat. + * + * Open a session. + */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + memset(&hs_key, 0, sizeof(hs_key)); + memset(&hs_value, 0, sizeof(hs_value)); + hs_start_ts = 0; /* [-Wconditional-uninitialized] */ + hs_counter = 0; /* [-Wconditional-uninitialized] */ + hs_btree_id = 0; /* [-Wconditional-uninitialized] */ + for (restart = true;;) { + /* + * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. + */ + while ((ret = session->open_cursor(session, WT_HS_URI, NULL, NULL, &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); + + /* + * The history file has mostly tombstones, ignore them and retrieve the underlying values. + * We don't care about tombstones, but we do want to hit every key rather than skip over + * them. This is a rollback-to-stable flag we're using for our own purposes. + */ + F_SET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); + + /* Search to the last-known location. */ + if (!restart) { + cursor->set_key(cursor, hs_btree_id, &hs_key, hs_start_ts, hs_counter); + ret = cursor->search_near(cursor, &exact); + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + } + + /* Get some more key/value pairs. */ + for (i = mmrand(NULL, 0, 1000); i > 0; --i) { + if ((ret = cursor->next(cursor)) == 0) { + testutil_check( + cursor->get_key(cursor, &hs_btree_id, &hs_key, &hs_start_ts, &hs_counter)); + testutil_check(cursor->get_value( + cursor, &hs_stop_durable_ts, &hs_durable_timestamp, &hs_upd_type, &hs_value)); + continue; + } + testutil_assert(ret == WT_NOTFOUND); + break; + } + + /* + * If we didn't hit the end of the store, save the current key to continue in the next run. + * Otherwise, reset so we'll start over. + */ + if (ret == 0) { + testutil_check(__wt_buf_grow(CUR2S(cursor), &hs_key, hs_key.size)); + restart = false; + } else + restart = true; + + testutil_check(cursor->close(cursor)); + + /* Sleep for some number of seconds, in short intervals so we don't make the run wait. */ + for (period = mmrand(NULL, 1, 10); period > 0 && !g.workers_finished; --period) + __wt_sleep(1, 0); + if (g.workers_finished) + break; + } + + testutil_check(session->close(session, NULL)); + + return (WT_THREAD_RET_VALUE); +} diff --git a/src/third_party/wiredtiger/test/format/kv.c b/src/third_party/wiredtiger/test/format/kv.c index 6305376eaa8..c6e1db1f4a3 100644 --- a/src/third_party/wiredtiger/test/format/kv.c +++ b/src/third_party/wiredtiger/test/format/kv.c @@ -51,7 +51,7 @@ key_init(void) if ((fp = fopen(g.home_key, "r")) == NULL) testutil_die(errno, "%s", g.home_key); for (i = 0; i < WT_ELEMENTS(g.key_rand_len); ++i) - fp_readv(fp, g.home_key, false, &g.key_rand_len[i]); + fp_readv(fp, g.home_key, &g.key_rand_len[i]); fclose_and_clear(&fp); return; } diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 7d7a9b1fa21..ac77565170e 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -38,6 +38,7 @@ static int col_update(TINFO *, WT_CURSOR *, bool); static int nextprev(TINFO *, WT_CURSOR *, bool); static WT_THREAD_RET ops(void *); static int read_row(TINFO *, WT_CURSOR *); +static int read_row_worker(WT_CURSOR *, TINFO *, uint64_t, WT_ITEM *, WT_ITEM *, bool); static int row_insert(TINFO *, WT_CURSOR *, bool); static int row_modify(TINFO *, WT_CURSOR *, bool); static int row_remove(TINFO *, WT_CURSOR *, bool); @@ -110,7 +111,7 @@ operations(u_int ops_seconds, bool lastrun) TINFO *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, random_tid; + wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, hs_tid, random_tid; wt_thread_t timestamp_tid; int64_t fourths, quit_fourths, thread_ops; uint32_t i; @@ -123,6 +124,7 @@ operations(u_int ops_seconds, bool lastrun) memset(&backup_tid, 0, sizeof(backup_tid)); memset(&checkpoint_tid, 0, sizeof(checkpoint_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); + memset(&hs_tid, 0, sizeof(hs_tid)); memset(&random_tid, 0, sizeof(random_tid)); memset(×tamp_tid, 0, sizeof(timestamp_tid)); @@ -151,15 +153,8 @@ operations(u_int ops_seconds, bool lastrun) quit_fourths = fourths + 15 * 4 * 60; } - /* - * We support replay of threaded runs, but don't log random numbers after threaded operations - * start, there's no point. - */ - if (!SINGLETHREADED) - g.rand_log_stop = true; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - logop(session, "%s", "=============== thread ops start"); + tracemsg("%s", "=============== thread ops start"); /* Initialize locks to single-thread backups, failures, and timestamp updates. */ lock_init(session, &g.backup_lock); @@ -174,18 +169,6 @@ operations(u_int ops_seconds, bool lastrun) tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); tinfo->id = (int)i + 1; - - /* - * Characterize the per-thread random number generator. Normally we want independent - * behavior so threads start in different parts of the RNG space, but we've found bugs by - * having the threads pound on the same key/value pairs, that is, by making them traverse - * the same RNG space. 75% of the time we run in independent RNG space. - */ - if (g.c_independent_thread_rng) - __wt_random_init_seed((WT_SESSION_IMPL *)session, &tinfo->rnd); - else - __wt_random_init(&tinfo->rnd); - tinfo->state = TINFO_RUNNING; testutil_check(__wt_thread_create(NULL, &tinfo->tid, ops, tinfo)); } @@ -199,6 +182,8 @@ operations(u_int ops_seconds, bool lastrun) testutil_check(__wt_thread_create(NULL, &backup_tid, backup, NULL)); if (g.c_compact) testutil_check(__wt_thread_create(NULL, &compact_tid, compact, NULL)); + if (g.c_hs_cursor) + testutil_check(__wt_thread_create(NULL, &hs_tid, hs_cursor, NULL)); if (g.c_random_cursor) testutil_check(__wt_thread_create(NULL, &random_tid, random_kv, NULL)); if (g.c_txn_timestamps) @@ -281,6 +266,8 @@ operations(u_int ops_seconds, bool lastrun) testutil_check(__wt_thread_join(NULL, &checkpoint_tid)); if (g.c_compact) testutil_check(__wt_thread_join(NULL, &compact_tid)); + if (g.c_hs_cursor) + testutil_check(__wt_thread_join(NULL, &hs_tid)); if (g.c_random_cursor) testutil_check(__wt_thread_join(NULL, &random_tid)); if (g.c_txn_timestamps) @@ -290,7 +277,7 @@ operations(u_int ops_seconds, bool lastrun) lock_destroy(session, &g.backup_lock); lock_destroy(session, &g.ts_lock); - logop(session, "%s", "=============== thread ops stop"); + tracemsg("%s", "=============== thread ops stop"); testutil_check(session->close(session, NULL)); for (i = 0; i < g.c_threads; ++i) { @@ -347,7 +334,7 @@ begin_transaction_ts(TINFO *tinfo, u_int *iso_configp) ret = session->timestamp_transaction(session, buf); if (ret == 0) { snap_init(tinfo, ts, true); - logop(session, "begin snapshot read-ts=%" PRIu64 " (repeatable)", ts); + traceop(tinfo, "begin snapshot read-ts=%" PRIu64 " (repeatable)", ts); return; } if (ret != EINVAL) @@ -375,7 +362,7 @@ begin_transaction_ts(TINFO *tinfo, u_int *iso_configp) lock_writeunlock(session, &g.ts_lock); snap_init(tinfo, ts, false); - logop(session, "begin snapshot read-ts=%" PRIu64 " (not repeatable)", ts); + traceop(tinfo, "begin snapshot read-ts=%" PRIu64 " (not repeatable)", ts); } /* @@ -416,7 +403,7 @@ begin_transaction(TINFO *tinfo, u_int *iso_configp) wiredtiger_begin_transaction(session, config); snap_init(tinfo, WT_TS_NONE, false); - logop(session, "begin %s", log); + traceop(tinfo, "begin %s", log); } /* @@ -455,8 +442,8 @@ commit_transaction(TINFO *tinfo, bool prepared) /* Remember our oldest commit timestamp. */ tinfo->commit_ts = ts; - logop( - session, "commit read-ts=%" PRIu64 ", commit-ts=%" PRIu64, tinfo->read_ts, tinfo->commit_ts); + traceop( + tinfo, "commit read-ts=%" PRIu64 ", commit-ts=%" PRIu64, tinfo->read_ts, tinfo->commit_ts); } /* @@ -474,7 +461,7 @@ rollback_transaction(TINFO *tinfo) testutil_check(session->rollback_transaction(session, NULL)); - logop(session, "abort read-ts=%" PRIu64, tinfo->read_ts); + traceop(tinfo, "abort read-ts=%" PRIu64, tinfo->read_ts); } /* @@ -510,7 +497,7 @@ prepare_transaction(TINFO *tinfo) testutil_check(__wt_snprintf(buf, sizeof(buf), "prepare_timestamp=%" PRIx64, ts)); ret = session->prepare_transaction(session, buf); - logop(session, "prepare ts=%" PRIu64, ts); + traceop(tinfo, "prepare ts=%" PRIu64, ts); lock_writeunlock(session, &g.ts_lock); @@ -616,6 +603,18 @@ ops(void *arg) bool greater_than, intxn, next, positioned, prepared; tinfo = arg; + testutil_check(__wt_thread_str(tinfo->tidbuf, sizeof(tinfo->tidbuf))); + + /* + * Characterize the per-thread random number generator. Normally we want independent behavior + * so threads start in different parts of the RNG space, but we've found bugs by having the + * threads pound on the same key/value pairs, that is, by making them traverse the same RNG + * space. 75% of the time we run in independent RNG space. + */ + if (g.c_independent_thread_rng) + __wt_random_init_seed(NULL, &tinfo->rnd); + else + __wt_random_init(&tinfo->rnd); iso_config = ISOLATION_RANDOM; /* -Wconditional-uninitialized */ @@ -641,6 +640,9 @@ ops(void *arg) /* Set the first operation where we'll truncate a range. */ truncate_op = g.c_truncate == 0 ? UINT64_MAX : mmrand(&tinfo->rnd, 100, 10000); + /* Initialize operation trace. */ + trace_ops_init(tinfo); + for (intxn = false; !tinfo->quit; ++tinfo->ops) { /* Periodically open up a new session and cursors. */ if (tinfo->ops > session_op || session == NULL || cursor == NULL) { @@ -1054,7 +1056,7 @@ wts_read_scan(void) if (keyno > g.rows) keyno = g.rows; - switch (ret = read_row_worker(cursor, keyno, &key, &value, false)) { + switch (ret = read_row_worker(cursor, NULL, keyno, &key, &value, false)) { case 0: case WT_NOTFOUND: case WT_ROLLBACK: @@ -1075,15 +1077,13 @@ wts_read_scan(void) * read_row_worker -- * Read and verify a single element in a row- or column-store file. */ -int -read_row_worker(WT_CURSOR *cursor, uint64_t keyno, WT_ITEM *key, WT_ITEM *value, bool sn) +static int +read_row_worker( + WT_CURSOR *cursor, TINFO *tinfo, uint64_t keyno, WT_ITEM *key, WT_ITEM *value, bool sn) { - WT_SESSION *session; uint8_t bitfield; int exact, ret; - session = cursor->session; - /* Retrieve the key/value pair by key. */ switch (g.type) { case FIX: @@ -1131,12 +1131,21 @@ read_row_worker(WT_CURSOR *cursor, uint64_t keyno, WT_ITEM *key, WT_ITEM *value, if (ret == 0) switch (g.type) { case FIX: - logop(session, "%-10s%" PRIu64 " {0x%02x}", "read", keyno, ((char *)value->data)[0]); + if (tinfo == NULL && g.trace_all) + tracemsg("%-10s%" PRIu64 " {0x%02x}", "read", keyno, ((char *)value->data)[0]); + if (tinfo != NULL) + traceop( + tinfo, "%-10s%" PRIu64 " {0x%02x}", "read", keyno, ((char *)value->data)[0]); + break; case ROW: case VAR: - logop(session, "%-10s%" PRIu64 " {%.*s}", "read", keyno, (int)value->size, - (char *)value->data); + if (tinfo == NULL && g.trace_all) + tracemsg( + "%-10s%" PRIu64 " {%.*s}", "read", keyno, (int)value->size, (char *)value->data); + if (tinfo != NULL) + traceop(tinfo, "%-10s%" PRIu64 " {%.*s}", "read", keyno, (int)value->size, + (char *)value->data); break; } @@ -1152,7 +1161,7 @@ read_row(TINFO *tinfo, WT_CURSOR *cursor) { /* 25% of the time we call search-near. */ return (read_row_worker( - cursor, tinfo->keyno, tinfo->key, tinfo->value, mmrand(&tinfo->rnd, 0, 3) == 1)); + cursor, tinfo, tinfo->keyno, tinfo->key, tinfo->value, mmrand(&tinfo->rnd, 0, 3) == 1)); } /* @@ -1272,19 +1281,18 @@ order_error_row: return (ret); } - if (ret == 0) + if (g.trace_all && ret == 0) switch (g.type) { case FIX: - logop( - cursor->session, "%-10s%" PRIu64 " {0x%02x}", which, keyno, ((char *)value.data)[0]); + traceop(tinfo, "%-10s%" PRIu64 " {0x%02x}", which, keyno, ((char *)value.data)[0]); break; case ROW: - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", which, keyno, (int)key.size, + traceop(tinfo, "%-10s%" PRIu64 " {%.*s}, {%.*s}", which, keyno, (int)key.size, (char *)key.data, (int)value.size, (char *)value.data); break; case VAR: - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", which, keyno, (int)value.size, - (char *)value.data); + traceop( + tinfo, "%-10s%" PRIu64 " {%.*s}", which, keyno, (int)value.size, (char *)value.data); break; } @@ -1308,8 +1316,8 @@ row_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) if ((ret = cursor->reserve(cursor)) != 0) return (ret); - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", "reserve", tinfo->keyno, - (int)tinfo->key->size, (char *)tinfo->key->data); + traceop(tinfo, "%-10s%" PRIu64 " {%.*s}", "reserve", tinfo->keyno, (int)tinfo->key->size, + (char *)tinfo->key->data); return (0); } @@ -1329,7 +1337,7 @@ col_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) if ((ret = cursor->reserve(cursor)) != 0) return (ret); - logop(cursor->session, "%-10s%" PRIu64, "reserve", tinfo->keyno); + traceop(tinfo, "%-10s%" PRIu64, "reserve", tinfo->keyno); return (0); } @@ -1380,9 +1388,8 @@ row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) testutil_check(cursor->get_value(cursor, tinfo->value)); - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "modify", tinfo->keyno, - (int)tinfo->key->size, (char *)tinfo->key->data, (int)tinfo->value->size, - (char *)tinfo->value->data); + traceop(tinfo, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "modify", tinfo->keyno, (int)tinfo->key->size, + (char *)tinfo->key->data, (int)tinfo->value->size, (char *)tinfo->value->data); return (0); } @@ -1407,8 +1414,8 @@ col_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) testutil_check(cursor->get_value(cursor, tinfo->value)); - logop(cursor->session, "%-10s%" PRIu64 ", {%.*s}", "modify", tinfo->keyno, - (int)tinfo->value->size, (char *)tinfo->value->data); + traceop(tinfo, "%-10s%" PRIu64 ", {%.*s}", "modify", tinfo->keyno, (int)tinfo->value->size, + (char *)tinfo->value->data); return (0); } @@ -1455,7 +1462,7 @@ row_truncate(TINFO *tinfo, WT_CURSOR *cursor) if (ret != 0) return (ret); - logop(session, "%-10s%" PRIu64 ", %" PRIu64, "truncate", tinfo->keyno, tinfo->last); + traceop(tinfo, "%-10s%" PRIu64 ", %" PRIu64, "truncate", tinfo->keyno, tinfo->last); return (0); } @@ -1497,7 +1504,7 @@ col_truncate(TINFO *tinfo, WT_CURSOR *cursor) if (ret != 0) return (ret); - logop(session, "%-10s%" PRIu64 "-%" PRIu64, "truncate", tinfo->keyno, tinfo->last); + traceop(tinfo, "%-10s%" PRIu64 "-%" PRIu64, "truncate", tinfo->keyno, tinfo->last); return (0); } @@ -1521,9 +1528,8 @@ row_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) if ((ret = cursor->update(cursor)) != 0) return (ret); - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "update", tinfo->keyno, - (int)tinfo->key->size, (char *)tinfo->key->data, (int)tinfo->value->size, - (char *)tinfo->value->data); + traceop(tinfo, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "update", tinfo->keyno, (int)tinfo->key->size, + (char *)tinfo->key->data, (int)tinfo->value->size, (char *)tinfo->value->data); return (0); } @@ -1549,11 +1555,11 @@ col_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) return (ret); if (g.type == FIX) - logop(cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "update", tinfo->keyno, + traceop(tinfo, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "update", tinfo->keyno, ((uint8_t *)tinfo->value->data)[0]); else - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", "update", tinfo->keyno, - (int)tinfo->value->size, (char *)tinfo->value->data); + traceop(tinfo, "%-10s%" PRIu64 " {%.*s}", "update", tinfo->keyno, (int)tinfo->value->size, + (char *)tinfo->value->data); return (0); } @@ -1582,9 +1588,8 @@ row_insert(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) return (ret); /* Log the operation */ - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "insert", tinfo->keyno, - (int)tinfo->key->size, (char *)tinfo->key->data, (int)tinfo->value->size, - (char *)tinfo->value->data); + traceop(tinfo, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "insert", tinfo->keyno, (int)tinfo->key->size, + (char *)tinfo->key->data, (int)tinfo->value->size, (char *)tinfo->value->data); return (0); } @@ -1669,11 +1674,11 @@ col_insert(TINFO *tinfo, WT_CURSOR *cursor) col_insert_add(tinfo); /* Extend the object. */ if (g.type == FIX) - logop(cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "insert", tinfo->keyno, + traceop(tinfo, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "insert", tinfo->keyno, ((uint8_t *)tinfo->value->data)[0]); else - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", "insert", tinfo->keyno, - (int)tinfo->value->size, (char *)tinfo->value->data); + traceop(tinfo, "%-10s%" PRIu64 " {%.*s}", "insert", tinfo->keyno, (int)tinfo->value->size, + (char *)tinfo->value->data); return (0); } @@ -1699,7 +1704,7 @@ row_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) if (ret != 0 && ret != WT_NOTFOUND) return (ret); - logop(cursor->session, "%-10s%" PRIu64, "remove", tinfo->keyno); + traceop(tinfo, "%-10s%" PRIu64, "remove", tinfo->keyno); return (ret); } @@ -1723,7 +1728,7 @@ col_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) if (ret != 0 && ret != WT_NOTFOUND) return (ret); - logop(cursor->session, "%-10s%" PRIu64, "remove", tinfo->keyno); + traceop(tinfo, "%-10s%" PRIu64, "remove", tinfo->keyno); return (ret); } diff --git a/src/third_party/wiredtiger/test/format/rebalance.c b/src/third_party/wiredtiger/test/format/rebalance.c index ea5d38a6d16..c17540d1a18 100644 --- a/src/third_party/wiredtiger/test/format/rebalance.c +++ b/src/third_party/wiredtiger/test/format/rebalance.c @@ -50,23 +50,17 @@ wts_rebalance(void) testutil_check(__wt_snprintf(cmd, len, REBALANCE_COPY_CMD, g.home, g.home, "orig", g.uri)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); - /* Rebalance, then verify the object. */ - wts_reopen(); - conn = g.wts_conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - logop(session, "%s", "=============== rebalance start"); + /* Open the database, rebalance and verify the object, then close the database. */ + wts_open(g.home, &conn, &session, true); + testutil_check(session->rebalance(session, g.uri, NULL)); + testutil_check(session->verify(session, g.uri, "strict")); + wts_close(&conn, &session); - testutil_checkfmt(session->rebalance(session, g.uri, NULL), "%s", g.uri); - - logop(session, "%s", "=============== rebalance stop"); - testutil_check(session->close(session, NULL)); - - wts_verify("post-rebalance verify"); - wts_close(); + /* Dump the rebalanced object. */ testutil_check(__wt_snprintf(cmd, len, REBALANCE_COPY_CMD, g.home, g.home, "new", g.uri)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); - /* Compare the old/new versions of the object. */ + /* Compare the old/new dumps of the object. */ testutil_check(__wt_snprintf(cmd, len, REBALANCE_CMP_CMD, g.home, g.home)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index 8d1a2e7ed0f..8f8d3c12786 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -29,24 +29,6 @@ #include "format.h" /* - * salvage -- - * A single salvage. - */ -static void -salvage(void) -{ - WT_CONNECTION *conn; - WT_SESSION *session; - - conn = g.wts_conn; - track("salvage", 0ULL, NULL); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->salvage(session, g.uri, "force=true")); - testutil_check(session->close(session, NULL)); -} - -/* * corrupt -- * Corrupt the file in a random way. */ @@ -135,13 +117,17 @@ found: void wts_salvage(void) { + WT_CONNECTION *conn; WT_DECL_RET; + WT_SESSION *session; size_t len; char *cmd; if (g.c_salvage == 0) return; + track("salvage", 0ULL, NULL); + /* Save a copy of the interesting files so we can replay the salvage step as necessary. */ len = strlen(g.home) + strlen(SALVAGE_COPY_CMD) + 1; cmd = dmalloc(len); @@ -151,17 +137,18 @@ wts_salvage(void) free(cmd); /* Salvage, then verify. */ - wts_open(g.home, true, &g.wts_conn, true); - salvage(); + wts_open(g.home, &conn, &session, true); + testutil_check(session->salvage(session, g.uri, "force=true")); + #if 0 wts_verify("post-salvage verify"); #endif - wts_close(); + wts_close(&conn, &session); /* Corrupt the file randomly, salvage, then verify. */ if (corrupt()) { - wts_open(g.home, true, &g.wts_conn, false); - salvage(); - wts_close(); + wts_open(g.home, &conn, &session, false); + testutil_check(session->salvage(session, g.uri, "force=true")); + wts_close(&conn, &session); } } diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index 35fcf20791f..031f52635d6 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -245,7 +245,7 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) #if WIREDTIGER_VERSION_MAJOR >= 10 testutil_check(__wt_debug_cursor_tree_hs(cursor, g.home_hsdump)); #endif - if (g.logging) + if (g.c_logging) testutil_check(cursor->session->log_flush(cursor->session, "sync=off")); #endif @@ -503,7 +503,7 @@ snap_repeat_single(WT_CURSOR *cursor, TINFO *tinfo) ret = session->timestamp_transaction(session, buf); if (ret == 0) { - logop(session, "%-10s%" PRIu64 " ts=%" PRIu64 " {%.*s}", "repeat", snap->keyno, snap->ts, + traceop(tinfo, "%-10s%" PRIu64 " ts=%" PRIu64 " {%.*s}", "repeat", snap->keyno, snap->ts, (int)snap->vsize, (char *)snap->vdata); /* The only expected error is rollback. */ diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 61d612b1abd..d363c037ae7 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -164,7 +164,7 @@ main(int argc, char *argv[]) /* Set values from the command line. */ home = NULL; one_flag = quiet_flag = false; - while ((ch = __wt_getopt(progname, argc, argv, "1BC:c:h:lqRrt:")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "1BC:c:h:qRrT:t")) != EOF) switch (ch) { case '1': /* One run and quit */ one_flag = true; @@ -181,17 +181,20 @@ main(int argc, char *argv[]) case 'h': home = __wt_optarg; break; - case 'l': /* Log operations to a file */ - g.logging = true; - break; case 'q': /* Quiet */ quiet_flag = true; break; case 'R': /* Reopen (start running on an existing database) */ g.reopen = true; break; - case 'r': /* Replay a run (use the configuration and random numbers from a previous run) */ - g.replay = true; + case 'T': /* Trace specifics. */ + if (trace_config(__wt_optarg) != 0) { + fprintf(stderr, "unexpected trace configuration \"%s\"\n", __wt_optarg); + usage(); + } + /* FALLTHROUGH */ + case 't': /* Trace */ + g.trace = true; break; default: usage(); @@ -202,15 +205,15 @@ main(int argc, char *argv[]) path_setup(home); /* - * If it's a replay or a reopen, use the already existing home directory's CONFIG file. + * If it's a reopen, use the already existing home directory's CONFIG file. * * If we weren't given a configuration file, set values from "CONFIG", if it exists. Small hack * to ignore any CONFIG file named ".", that just makes it possible to ignore any local CONFIG * file, used when running checks. */ - if (g.reopen || g.replay) { + if (g.reopen) { if (config != NULL) - testutil_die(EINVAL, "-c incompatible with -R or -r"); + testutil_die(EINVAL, "-c incompatible with -R"); if (access(g.home_config, R_OK) != 0) testutil_die(ENOENT, "%s", g.home_config); config = g.home_config; @@ -238,18 +241,6 @@ main(int argc, char *argv[]) g.c_quiet = 1; /* - * Multithreaded runs can be replayed: it's useful and we'll get the configuration correct. - * Obviously the order of operations changes, warn the user. - * - * Single-threaded runs historically exited after a single replay, which makes sense when you're - * debugging, leave that semantic in place. - */ - if (g.replay && !SINGLETHREADED) - printf("Warning: replaying a multi-threaded run\n"); - if (g.replay && SINGLETHREADED) - g.c_runs = 1; - - /* * Calculate how long each operations loop should run. Take any timer value and convert it to * seconds, then allocate 15 seconds to do initialization, verification, rebalance and/or * salvage tasks after the operations loop finishes. This is not intended to be exact in any @@ -263,24 +254,25 @@ main(int argc, char *argv[]) __wt_random_init_seed(NULL, &g.rnd); /* Initialize the RNG. */ + testutil_check(__wt_thread_str(g.tidbuf, sizeof(g.tidbuf))); + printf("%s: process %" PRIdMAX " running\n", progname, (intmax_t)getpid()); fflush(stdout); while (++g.run_cnt <= g.c_runs || g.c_runs == 0) { __wt_seconds(NULL, &start); track("starting up", 0ULL, NULL); - if (!g.reopen) - wts_create(); /* Create and initialize the database and an object. */ - - config_final(); /* Remaining configuration and validation */ + config_run(); - handle_init(); + if (g.reopen) { + config_final(); + wts_open(g.home, &g.wts_conn, NULL, true); + } else { + wts_create(g.home); + config_final(); + wts_open(g.home, &g.wts_conn, NULL, true); + trace_init(); - if (g.reopen) - wts_reopen(); /* Reopen existing database. */ - else { - wts_open(g.home, true, &g.wts_conn, true); - wts_init(); TIMED_MAJOR_OP(wts_load()); /* Load and verify initial records */ TIMED_MAJOR_OP(wts_verify("post-bulk verify")); } @@ -303,7 +295,7 @@ main(int argc, char *argv[]) TIMED_MAJOR_OP(wts_verify("post-ops verify")); track("shutting down", 0ULL, NULL); - wts_close(); + wts_close(&g.wts_conn, NULL); /* * Rebalance testing. @@ -315,7 +307,7 @@ main(int argc, char *argv[]) */ TIMED_MAJOR_OP(wts_salvage()); - handle_teardown(); + trace_teardown(); /* Overwrite the progress line with a completion line. */ if (!g.c_quiet) @@ -344,21 +336,19 @@ format_die(void) { /* - * Turn off tracking and logging so we don't obscure the error message. The lock we're about to - * acquire will act as a barrier to flush the writes. This is really a "best effort" more than a - * guarantee, there's too much stuff in flight to be sure. + * Turn off progress reports and logging so we don't obscure the error message. The lock we're + * about to acquire will act as a barrier to flush the writes. This is really a "best effort" + * more than a guarantee, there's too much stuff in flight to be sure. */ g.c_quiet = 1; - g.logging = false; + g.trace = false; /* * Single-thread error handling, our caller exits after calling us (we never release the lock). */ (void)pthread_rwlock_wrlock(&g.death_lock); - /* Flush/close any logging information. */ - fclose_and_clear(&g.logfp); - fclose_and_clear(&g.randfp); + trace_teardown(); fprintf(stderr, "\n%s: run FAILED\n", progname); @@ -375,8 +365,8 @@ static void usage(void) { fprintf(stderr, - "usage: %s [-1BlqRr] [-C wiredtiger-config]\n " - "[-c config-file] [-h home] [name=value ...]\n", + "usage: %s [-1BqRt] [-C wiredtiger-config]\n " + "[-c config-file] [-h home] [-T trace-options] [name=value ...]\n", progname); fprintf(stderr, "%s", "\t-1 run once then quit\n" @@ -384,10 +374,10 @@ usage(void) "\t-C specify wiredtiger_open configuration arguments\n" "\t-c read test program configuration from a file (default 'CONFIG')\n" "\t-h home directory (default 'RUNDIR')\n" - "\t-l log operations to a file\n" "\t-q run quietly\n" "\t-R run on an existing database\n" - "\t-r replay the last run from the home directory configuration\n"); + "\t-T all|local\n" + "\t-t log operations\n"); config_error(); exit(EXIT_FAILURE); diff --git a/src/third_party/wiredtiger/test/format/trace.c b/src/third_party/wiredtiger/test/format/trace.c new file mode 100644 index 00000000000..11f0c6ae3aa --- /dev/null +++ b/src/third_party/wiredtiger/test/format/trace.c @@ -0,0 +1,134 @@ +/*- + * Public Domain 2014-2020 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "format.h" + +#define TRACE_DIR "OPS.TRACE" +#define TRACE_INIT_CMD "rm -rf %s/" TRACE_DIR " && mkdir %s/" TRACE_DIR + +int +trace_config(const char *config) +{ + WT_DECL_RET; + char *copy, *p; + + copy = dstrdup(config); + for (;;) { + if ((p = strstr(copy, "all")) != NULL) { + g.trace_all = true; + memset(p, ' ', strlen("all")); + continue; + } + if ((p = strstr(copy, "local")) != NULL) { + g.trace_local = true; + memset(p, ' ', strlen("local")); + continue; + } + break; + } + + for (p = copy; *p != '\0'; ++p) + if (*p != ',' && !__wt_isspace((u_char)*p)) { + ret = EINVAL; + break; + } + + free(copy); + return (ret); +} + +void +trace_init(void) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + size_t len; + char *p; + const char *config; + + if (!g.trace) + return; + + /* Write traces to a separate database by default, optionally write traces to the primary. */ + if (g.trace_local) { + if (!g.c_logging) + testutil_die(EINVAL, + "operation logging to the primary database requires logging be configured for that " + "database"); + + conn = g.wts_conn; + + /* Keep the last N log files. */ + testutil_check(conn->reconfigure(conn, "debug_mode=(log_retention=10)")); + } else { + len = strlen(g.home) * 2 + strlen(TRACE_INIT_CMD) + 10; + p = dmalloc(len); + testutil_check(__wt_snprintf(p, len, TRACE_INIT_CMD, g.home, g.home)); + testutil_checkfmt(system(p), "%s", "logging directory creation failed"); + free(p); + + /* Configure logging with archival, and keep the last N log files. */ + len = strlen(g.home) * strlen(TRACE_DIR) + 10; + p = dmalloc(len); + testutil_check(__wt_snprintf(p, len, "%s/%s", g.home, TRACE_DIR)); + config = "create,log=(enabled,archive),debug_mode=(log_retention=10)"; + testutil_checkfmt(wiredtiger_open(p, NULL, config, &conn), "%s: %s", p, config); + free(p); + } + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + g.trace_conn = conn; + g.trace_session = session; +} + +void +trace_teardown(void) +{ + WT_CONNECTION *conn; + + conn = g.trace_conn; + g.trace_conn = NULL; + + if (!g.trace || g.trace_local || conn == NULL) + return; + + testutil_check(conn->close(conn, NULL)); +} + +void +trace_ops_init(TINFO *tinfo) +{ + WT_SESSION *session; + + if (!g.trace) + return; + + testutil_check(g.trace_conn->open_session(g.trace_conn, NULL, NULL, &session)); + tinfo->trace = session; +} diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 7aba99c20de..c8be232e1d5 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -102,12 +102,6 @@ path_setup(const char *home) g.home_rand = dmalloc(len); testutil_check(__wt_snprintf(g.home_rand, len, "%s/%s", g.home, name)); - /* Log file. */ - name = "OPERATIONS.log"; - len = strlen(g.home) + strlen(name) + 2; - g.home_log = dmalloc(len); - testutil_check(__wt_snprintf(g.home_log, len, "%s/%s", g.home, name)); - /* History store dump file. */ name = "FAIL.HSdump"; len = strlen(g.home) + strlen(name) + 2; @@ -132,19 +126,13 @@ path_setup(const char *home) * Read and return a value from a file. */ bool -fp_readv(FILE *fp, char *name, bool eof_ok, uint32_t *vp) +fp_readv(FILE *fp, char *name, uint32_t *vp) { u_long ulv; char *endptr, buf[100]; - if (fgets(buf, sizeof(buf), fp) == NULL) { - if (feof(g.randfp)) { - if (eof_ok) - return (true); - testutil_die(errno, "%s: read-value EOF", name); - } + if (fgets(buf, sizeof(buf), fp) == NULL) testutil_die(errno, "%s: read-value error", name); - } errno = 0; ulv = strtoul(buf, &endptr, 10); @@ -155,64 +143,6 @@ fp_readv(FILE *fp, char *name, bool eof_ok, uint32_t *vp) } /* - * rng_slow -- - * Return a random number, doing the real work. - */ -uint32_t -rng_slow(WT_RAND_STATE *rnd) -{ - uint32_t v; - - /* - * We can reproduce a single-threaded run based on the random numbers used in the initial run, - * plus the configuration files. - */ - if (g.replay) { - if (fp_readv(g.randfp, g.home_rand, true, &v)) { - fprintf(stderr, - "\n" - "end of random number log reached\n"); - exit(EXIT_SUCCESS); - } - return (v); - } - - v = __wt_random(rnd); - - /* Save and flush the random number so we're up-to-date on error. */ - (void)fprintf(g.randfp, "%" PRIu32 "\n", v); - (void)fflush(g.randfp); - - return (v); -} - -/* - * handle_init -- - * Initialize logging/random number handles for a run. - */ -void -handle_init(void) -{ - /* Open/truncate logging/random number handles. */ - if (g.logging && (g.logfp = fopen(g.home_log, "w")) == NULL) - testutil_die(errno, "fopen: %s", g.home_log); - if ((g.randfp = fopen(g.home_rand, g.replay ? "r" : "w")) == NULL) - testutil_die(errno, "%s", g.home_rand); -} - -/* - * handle_teardown -- - * Shutdown logging/random number handles for a run. - */ -void -handle_teardown(void) -{ - /* Flush/close logging/random number handles. */ - fclose_and_clear(&g.logfp); - fclose_and_clear(&g.randfp); -} - -/* * fclose_and_clear -- * Close a file and clear the handle so we don't close twice. */ diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 309b576be63..91b8c5051aa 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -40,25 +40,6 @@ "cd %s > /dev/null && rm -rf `ls | sed /CONFIG.rand/d`" /* - * wts_create -- - * Create the database home. - */ -void -wts_create(void) -{ - WT_DECL_RET; - size_t len; - char *cmd; - - len = strlen(g.home) * 3 + strlen(FORMAT_HOME_INIT_CMD) + 1; - cmd = dmalloc(len); - testutil_check(__wt_snprintf(cmd, len, FORMAT_HOME_INIT_CMD, g.home, g.home, g.home)); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "home initialization (\"%s\") failed", cmd); - free(cmd); -} - -/* * compressor -- * Configure compression. */ @@ -118,7 +99,8 @@ encryptor(uint32_t encrypt_flag) static int handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { - int out; + WT_DECL_RET; + int nw; (void)(handler); (void)(session); @@ -132,20 +114,11 @@ handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *messa return (0); /* Write and flush the message so we're up-to-date on error. */ - if (g.logfp == NULL) { - out = printf("%p:%s\n", (void *)session, message); - (void)fflush(stdout); - } else { - out = fprintf(g.logfp, "%p:%s\n", (void *)session, message); - (void)fflush(g.logfp); - } - return (out < 0 ? EIO : 0); + nw = printf("%p:%s\n", (void *)session, message); + ret = fflush(stdout); + return (nw < 0 ? EIO : (ret == EOF ? errno : 0)); } -/* - * __handle_progress_default -- - * Default WT_EVENT_HANDLER->handle_progress implementation: ignore. - */ static int handle_progress( WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *operation, uint64_t progress) @@ -172,20 +145,18 @@ static WT_EVENT_HANDLER event_handler = { } while (0) /* - * wts_open -- - * Open a connection to a WiredTiger database. + * create_database -- + * Create a WiredTiger database. */ -void -wts_open(const char *home, bool set_api, WT_CONNECTION **connp, bool allow_verify) +static void +create_database(const char *home, WT_CONNECTION **connp) { WT_CONNECTION *conn; size_t max; - char *config, *p; - - *connp = NULL; + char config[8 * 1024], *p; - config = p = g.wiredtiger_open_config; - max = sizeof(g.wiredtiger_open_config); + p = config; + max = sizeof(config); CONFIG_APPEND(p, "create=true" @@ -281,13 +252,6 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp, bool allow_verif CONFIG_APPEND(p, ",split_8"); CONFIG_APPEND(p, "]"); -#if WIREDTIGER_VERSION_MAJOR >= 10 - if (g.c_verify && allow_verify) - CONFIG_APPEND(p, ",verify_metadata=true"); - else if (!allow_verify) - CONFIG_APPEND(p, ",verify_metadata=false"); -#endif - /* Extensions. */ CONFIG_APPEND(p, ",extensions=[" @@ -311,42 +275,21 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp, bool allow_verif testutil_checkfmt(wiredtiger_open(home, &event_handler, config, &conn), "%s", home); - if (set_api) - g.wt_api = conn->get_extension_api(conn); - *connp = conn; } /* - * wts_reopen -- - * Re-open a connection to a WiredTiger database. - */ -void -wts_reopen(void) -{ - WT_CONNECTION *conn; - - testutil_checkfmt( - wiredtiger_open(g.home, &event_handler, g.wiredtiger_open_config, &conn), "%s", g.home); - - g.wt_api = conn->get_extension_api(conn); - g.wts_conn = conn; -} - -/* - * wts_init -- + * create_object -- * Create the database object. */ -void -wts_init(void) +static void +create_object(WT_CONNECTION *conn) { - WT_CONNECTION *conn; WT_SESSION *session; size_t max; uint32_t maxintlkey, maxleafkey, maxleafvalue; char config[4096], *p; - conn = g.wts_conn; p = config; max = sizeof(config); @@ -458,20 +401,89 @@ wts_init(void) testutil_check(session->close(session, NULL)); } +/* + * wts_create -- + * Create the database home and objects. + */ void -wts_close(void) +wts_create(const char *home) { WT_CONNECTION *conn; + WT_DECL_RET; + size_t len; + char *cmd; - conn = g.wts_conn; + len = strlen(g.home) * 3 + strlen(FORMAT_HOME_INIT_CMD) + 1; + cmd = dmalloc(len); + testutil_check(__wt_snprintf(cmd, len, FORMAT_HOME_INIT_CMD, g.home, g.home, g.home)); + if ((ret = system(cmd)) != 0) + testutil_die(ret, "home initialization (\"%s\") failed", cmd); + free(cmd); + + create_database(home, &conn); + create_object(conn); + if (g.c_in_memory != 0) + g.wts_conn_inmemory = conn; + else + testutil_check(conn->close(conn, NULL)); +} + +/* + * wts_open -- + * Open a connection to a WiredTiger database. + */ +void +wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool allow_verify) +{ + WT_CONNECTION *conn; + const char *config; + + *connp = NULL; + if (sessionp != NULL) + *sessionp = NULL; + + /* If in-memory, there's only a single, shared WT_CONNECTION handle. */ + if (g.c_in_memory != 0) + conn = g.wts_conn_inmemory; + else { + config = ""; +#if WIREDTIGER_VERSION_MAJOR >= 10 + if (g.c_verify && allow_verify) + config = ",verify_metadata=true"; +#else + WT_UNUSED(allow_verify); +#endif + testutil_checkfmt(wiredtiger_open(home, &event_handler, config, &conn), "%s", home); + } + + if (sessionp != NULL) + testutil_check(conn->open_session(conn, NULL, NULL, sessionp)); + *connp = conn; +} + +void +wts_close(WT_CONNECTION **connp, WT_SESSION **sessionp) +{ + WT_CONNECTION *conn; + + conn = *connp; + *connp = NULL; + + /* + * If running in-memory, there's only a single, shared WT_CONNECTION handle. Format currently + * doesn't perform the operations coded to close and then re-open the database on in-memory + * databases (for example, salvage or rebalance), so the close gets all references, it doesn't + * have to avoid closing the real handle. + */ + if (conn == g.wts_conn_inmemory) + g.wts_conn_inmemory = NULL; + if (sessionp != NULL) + *sessionp = NULL; if (g.backward_compatible) testutil_check(conn->reconfigure(conn, "compatibility=(release=3.3)")); testutil_check(conn->close(conn, g.c_leak_memory ? "leak_memory" : NULL)); - - g.wts_conn = NULL; - g.wt_api = NULL; } void @@ -488,7 +500,7 @@ wts_verify(const char *tag) track("verify", 0ULL, NULL); testutil_check(conn->open_session(conn, NULL, NULL, &session)); - logop(session, "%s", "=============== verify start"); + tracemsg("%s", "=============== verify start"); /* * Verify can return EBUSY if the handle isn't available. Don't yield and retry, in the case of @@ -497,7 +509,7 @@ wts_verify(const char *tag) ret = session->verify(session, g.uri, "strict"); testutil_assertfmt(ret == 0 || ret == EBUSY, "session.verify: %s: %s", g.uri, tag); - logop(session, "%s", "=============== verify stop"); + tracemsg("%s", "=============== verify stop"); testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/suite/test_prepare02.py b/src/third_party/wiredtiger/test/suite/test_prepare02.py index 315cf02329f..c10223940fe 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare02.py @@ -70,8 +70,6 @@ class test_prepare02(wttest.WiredTigerTestCase, suite_subprocess): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.log_flush("sync=on"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.log_printf("Printing to log file"), msg) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.rebalance("table:mytable", None), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.rename("table:mytable", "table:mynewtable", None), msg) |