diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2020-05-25 15:55:23 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-05-25 06:16:02 +0000 |
| commit | c480627eae1b5d09f8eea247b396b45b89a5c275 (patch) | |
| tree | 3f189d5d70cfab814fe7e0808692b271663d8c39 /src/third_party/wiredtiger | |
| parent | d169769bba283cd0a1906c81580ff8e7cf7bcd93 (diff) | |
| download | mongo-c480627eae1b5d09f8eea247b396b45b89a5c275.tar.gz | |
Import wiredtiger: 0ec94ad7e360d93cb91f558d7efc03c644b1a9f8 from branch mongodb-4.4
ref: 5a3c70d7e4..0ec94ad7e3
for: 4.5.1
WT-6199 Fix PPC test variables in evergreen.yml
WT-6219 Fix application threads history store visibility during eviction
WT-6220 Add statistics tracking when times are written to disk
WT-6251 Fix splits to reduce the size of index files
WT-6260 Disable post salvage verify in format
WT-6266 Metadata verify of the HS file can fail if underlying objects are corrupted
WT-6278 Return from resolve prepared op when we have already resolved the operations
WT-6284 History store isn't visible to wt dump
WT-6287 Remove WT_TXN_HAS_TS_READ flag
Diffstat (limited to 'src/third_party/wiredtiger')
29 files changed, 912 insertions, 248 deletions
diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 31b26f521d4..df6f1dc0c64 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -524,9 +524,10 @@ connection_stats = [ PerfHistStat('perf_hist_opwrite_latency_lt1000', 'operation write latency histogram (bucket 3) - 500-999us'), PerfHistStat('perf_hist_opwrite_latency_lt10000', 'operation write latency histogram (bucket 4) - 1000-9999us'), -########################################## + ########################################## # Reconciliation statistics ########################################## + RecStat('rec_maximum_seconds', 'maximum seconds spent in a reconciliation call', 'no_clear,no_scale,size'), RecStat('rec_page_delete', 'pages deleted'), RecStat('rec_page_delete_fast', 'fast-path pages deleted'), RecStat('rec_pages', 'page reconciliation calls'), @@ -536,6 +537,27 @@ connection_stats = [ RecStat('rec_pages_with_txn', 'page reconciliation calls that resulted in values with transaction ids'), RecStat('rec_split_stashed_bytes', 'split bytes currently awaiting free', 'no_clear,no_scale,size'), RecStat('rec_split_stashed_objects', 'split objects currently awaiting free', 'no_clear,no_scale'), + RecStat('rec_time_aggr_newest_start_durable_ts', 'pages written including an aggregated newest start durable timestamp '), + RecStat('rec_time_aggr_newest_stop_durable_ts', 'pages written including an aggregated newest stop durable timestamp '), + RecStat('rec_time_aggr_newest_stop_ts', 'pages written including an aggregated newest stop timestamp '), + RecStat('rec_time_aggr_newest_stop_txn', 'pages written including an aggregated newest stop transaction ID'), + RecStat('rec_time_aggr_oldest_start_ts', 'pages written including an aggregated oldest start timestamp '), + RecStat('rec_time_aggr_oldest_start_txn', 'pages written including an aggregated oldest start transaction ID '), + RecStat('rec_time_aggr_prepared', 'pages written including an aggregated prepare'), + RecStat('rec_time_window_durable_start_ts', 'records written including a start durable timestamp'), + RecStat('rec_time_window_durable_stop_ts', 'records written including a stop durable timestamp'), + RecStat('rec_time_window_pages_durable_start_ts', 'pages written including at least one start durable timestamp'), + RecStat('rec_time_window_pages_durable_stop_ts', 'pages written including at least one stop durable timestamp'), + RecStat('rec_time_window_pages_prepared', 'pages written including at least one prepare state'), + RecStat('rec_time_window_pages_start_ts', 'pages written including at least one start timestamp'), + RecStat('rec_time_window_pages_start_txn', 'pages written including at least one start transaction ID'), + RecStat('rec_time_window_pages_stop_ts', 'pages written including at least one stop timestamp'), + RecStat('rec_time_window_pages_stop_txn', 'pages written including at least one stop transaction ID'), + RecStat('rec_time_window_prepared', 'records written including a prepare state'), + RecStat('rec_time_window_start_ts', 'records written including a start timestamp'), + RecStat('rec_time_window_start_txn', 'records written including a start transaction ID'), + RecStat('rec_time_window_stop_ts', 'records written including a stop timestamp'), + RecStat('rec_time_window_stop_txn', 'records written including a stop transaction ID'), ########################################## # Session operations @@ -804,6 +826,13 @@ dsrc_stats = [ CursorStat('cursor_update_bytes_changed', 'update value size change', 'size'), ########################################## + # History statistics + ########################################## + HistoryStat('hs_gc_pages_evict', 'history pages added for eviction during garbage collection'), + HistoryStat('hs_gc_pages_removed', 'history pages removed for garbage collection'), + HistoryStat('hs_gc_pages_visited', 'history pages visited for garbage collection'), + + ########################################## # LSM statistics ########################################## LSMStat('bloom_count', 'bloom filters in the LSM tree', 'no_scale'), @@ -829,7 +858,6 @@ dsrc_stats = [ RecStat('rec_overflow_key_internal', 'internal-page overflow keys'), RecStat('rec_overflow_key_leaf', 'leaf-page overflow keys'), RecStat('rec_overflow_value', 'overflow values written'), - RecStat('rec_prepare_value', 'prepared values written'), RecStat('rec_page_delete', 'pages deleted'), RecStat('rec_page_delete_fast', 'fast-path pages deleted'), RecStat('rec_page_match', 'page checksum matches'), @@ -837,6 +865,27 @@ dsrc_stats = [ RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'), RecStat('rec_prefix_compression', 'leaf page key bytes discarded using prefix compression', 'size'), RecStat('rec_suffix_compression', 'internal page key bytes discarded using suffix compression', 'size'), + RecStat('rec_time_aggr_newest_start_durable_ts', 'pages written including an aggregated newest start durable timestamp '), + RecStat('rec_time_aggr_newest_stop_durable_ts', 'pages written including an aggregated newest stop durable timestamp '), + RecStat('rec_time_aggr_newest_stop_ts', 'pages written including an aggregated newest stop timestamp '), + RecStat('rec_time_aggr_newest_stop_txn', 'pages written including an aggregated newest stop transaction ID'), + RecStat('rec_time_aggr_oldest_start_ts', 'pages written including an aggregated oldest start timestamp '), + RecStat('rec_time_aggr_oldest_start_txn', 'pages written including an aggregated oldest start transaction ID '), + RecStat('rec_time_aggr_prepared', 'pages written including an aggregated prepare'), + RecStat('rec_time_window_durable_start_ts', 'records written including a start durable timestamp'), + RecStat('rec_time_window_durable_stop_ts', 'records written including a stop durable timestamp'), + RecStat('rec_time_window_pages_durable_start_ts', 'pages written including at least one start durable timestamp'), + RecStat('rec_time_window_pages_durable_stop_ts', 'pages written including at least one stop durable timestamp'), + RecStat('rec_time_window_pages_prepared', 'pages written including at least one prepare'), + RecStat('rec_time_window_pages_start_ts', 'pages written including at least one start timestamp'), + RecStat('rec_time_window_pages_start_txn', 'pages written including at least one start transaction ID'), + RecStat('rec_time_window_pages_stop_ts', 'pages written including at least one stop timestamp'), + RecStat('rec_time_window_pages_stop_txn', 'pages written including at least one stop transaction ID'), + RecStat('rec_time_window_prepared', 'records written including a prepare'), + RecStat('rec_time_window_start_ts', 'records written including a start timestamp'), + RecStat('rec_time_window_start_txn', 'records written including a start transaction ID'), + RecStat('rec_time_window_stop_ts', 'records written including a stop timestamp'), + RecStat('rec_time_window_stop_txn', 'records written including a stop transaction ID'), ########################################## # Session operations diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 62a34a3cad1..1346f75f0d1 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "5a3c70d7e4990685eedf765f6909bbbf4f824d5c" + "commit": "0ec94ad7e360d93cb91f558d7efc03c644b1a9f8" } diff --git a/src/third_party/wiredtiger/src/conn/conn_cache.c b/src/third_party/wiredtiger/src/conn/conn_cache.c index 0512d0f1b17..9c952020b95 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache.c @@ -317,6 +317,8 @@ __wt_cache_stats_update(WT_SESSION_IMPL *session) if (conn->evict_server_running) WT_STAT_SET(session, stats, cache_eviction_walks_active, cache->walk_session->nhazard); + WT_STAT_SET(session, stats, rec_maximum_seconds, conn->rec_maximum_seconds); + /* TODO: WT-5585 Remove lookaside score statistic after MongoDB switches to an alternative. */ WT_STAT_SET(session, stats, cache_lookaside_score, 0); } diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index ec38401950a..5ecdf6bcc38 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -200,7 +200,8 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32 /* * Track the largest page size seen at eviction, it tells us something about our ability to - * force pages out before they're larger than the cache. + * force pages out before they're larger than the cache. We don't care about races, it's just a + * statistic. */ if (page->memory_footprint > conn->cache->evict_max_page_size) conn->cache->evict_max_page_size = page->memory_footprint; diff --git a/src/third_party/wiredtiger/src/history/hs.c b/src/third_party/wiredtiger/src/history/hs.c index b1f5be9ed5e..bfec739ef5e 100644 --- a/src/third_party/wiredtiger/src/history/hs.c +++ b/src/third_party/wiredtiger/src/history/hs.c @@ -1021,7 +1021,7 @@ __wt_find_hs_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma * visibility checks when reading in order to construct the modify chain, so we can create * the value we expect. */ - F_SET(session, WT_SESSION_RESOLVING_MODIFY); + F_SET(session, WT_SESSION_HS_IGNORE_VISIBILITY); while (upd_type == WT_UPDATE_MODIFY) { WT_ERR(__wt_upd_alloc(session, hs_value, upd_type, &mod_upd, NULL)); WT_ERR(__wt_modify_vector_push(&modifies, mod_upd)); @@ -1064,7 +1064,7 @@ __wt_find_hs_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma &upd_type_full, hs_value)); upd_type = (uint8_t)upd_type_full; } - F_CLR(session, WT_SESSION_RESOLVING_MODIFY); + F_CLR(session, WT_SESSION_HS_IGNORE_VISIBILITY); WT_ASSERT(session, upd_type == WT_UPDATE_STANDARD); while (modifies.size > 0) { __wt_modify_vector_pop(&modifies, &mod_upd); @@ -1088,7 +1088,7 @@ skip_buf: done: err: - F_CLR(session, WT_SESSION_RESOLVING_MODIFY); + F_CLR(session, WT_SESSION_HS_IGNORE_VISIBILITY); if (orig_hs_value_buf != NULL) __wt_scr_free(session, &orig_hs_value_buf); @@ -1207,14 +1207,16 @@ __wt_hs_delete_key_from_ts( /* * In order to delete a key range, we need to be able to inspect all history store records - * regardless of their stop time points. + * regardless of their stop time points and the visibility of their values. */ F_SET(session->hs_cursor, WT_CURSTD_IGNORE_TOMBSTONE); + F_SET(session, WT_SESSION_HS_IGNORE_VISIBILITY); /* The tree structure can change while we try to insert the mod list, retry if that happens. */ while ((ret = __hs_delete_key_from_ts_int(session, btree_id, key, ts)) == WT_RESTART) ; + F_CLR(session, WT_SESSION_HS_IGNORE_VISIBILITY); F_CLR(session->hs_cursor, WT_CURSTD_IGNORE_TOMBSTONE); WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner)); diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 9c0ebf552be..4bf25bbb379 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -312,6 +312,7 @@ struct __wt_connection_impl { uint32_t stat_flags; /* Options declared in flags.py */ /* Connection statistics */ + uint64_t rec_maximum_seconds; /* Maximum seconds reconciliation took. */ WT_CONNECTION_STATS *stats[WT_COUNTER_SLOTS]; WT_CONNECTION_STATS *stat_array; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 6f48337e9d8..24931905227 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -502,4 +502,7 @@ struct __wt_cursor_table { #define WT_CURSOR_RECNO(cursor) WT_STREQ((cursor)->key_format, "r") +#define WT_CURSOR_IS_DUMP(cursor) \ + F_ISSET(cursor, (WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_DUMP_JSON)) + #define WT_CURSOR_RAW_OK (WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW) diff --git a/src/third_party/wiredtiger/src/include/reconcile.h b/src/third_party/wiredtiger/src/include/reconcile.h index 3b75373ae56..10f50a8c9b8 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.h +++ b/src/third_party/wiredtiger/src/include/reconcile.h @@ -144,6 +144,30 @@ struct __wt_reconcile { size_t min_space_avail; /* Remaining space in this chunk to put a minimum size boundary */ /* + * Counters tracking how much time information is included in reconciliation for each page that + * is written to disk. The number of entries on a page is limited to a 32 bit number so these + * counters can be too. + */ + uint32_t count_durable_start_ts; + uint32_t count_start_ts; + uint32_t count_start_txn; + uint32_t count_durable_stop_ts; + uint32_t count_stop_ts; + uint32_t count_stop_txn; + uint32_t count_prepare; + +/* AUTOMATIC FLAG VALUE GENERATION START */ +#define WT_REC_TIME_NEWEST_START_DURABLE_TS 0x01u +#define WT_REC_TIME_NEWEST_STOP_DURABLE_TS 0x02u +#define WT_REC_TIME_NEWEST_STOP_TS 0x04u +#define WT_REC_TIME_NEWEST_STOP_TXN 0x08u +#define WT_REC_TIME_OLDEST_START_TS 0x10u +#define WT_REC_TIME_OLDEST_START_TXN 0x20u +#define WT_REC_TIME_PREPARE 0x40u + /* AUTOMATIC FLAG VALUE GENERATION STOP */ + uint16_t ts_usage_flags; + + /* * Saved update list, supporting WT_REC_HS configurations. While reviewing updates for each * page, we save WT_UPDATE lists here, and then move them to per-block areas as the blocks are * defined. diff --git a/src/third_party/wiredtiger/src/include/reconcile.i b/src/third_party/wiredtiger/src/include/reconcile.i index 01aa284f0be..2bdb78d026f 100644 --- a/src/third_party/wiredtiger/src/include/reconcile.i +++ b/src/third_party/wiredtiger/src/include/reconcile.i @@ -13,27 +13,159 @@ (WT_CROSSING_MIN_BND(r, next_len) || WT_CROSSING_SPLIT_BND(r, next_len)) /* + * __rec_cell_addr_stats -- + * Track statistics for time values associated with an address. + */ +static inline void +__rec_cell_addr_stats(WT_RECONCILE *r, WT_TIME_AGGREGATE *ta) +{ + if (ta->newest_start_durable_ts != WT_TS_NONE) + FLD_SET(r->ts_usage_flags, WT_REC_TIME_NEWEST_START_DURABLE_TS); + if (ta->newest_stop_durable_ts != WT_TS_NONE) + FLD_SET(r->ts_usage_flags, WT_REC_TIME_NEWEST_STOP_DURABLE_TS); + if (ta->oldest_start_ts != WT_TS_NONE) + FLD_SET(r->ts_usage_flags, WT_REC_TIME_OLDEST_START_TS); + if (ta->oldest_start_txn != WT_TXN_NONE) + FLD_SET(r->ts_usage_flags, WT_REC_TIME_OLDEST_START_TXN); + if (ta->newest_stop_ts != WT_TS_MAX) + FLD_SET(r->ts_usage_flags, WT_REC_TIME_NEWEST_STOP_TS); + if (ta->newest_stop_txn != WT_TXN_MAX) + FLD_SET(r->ts_usage_flags, WT_REC_TIME_NEWEST_STOP_TXN); + if (ta->prepare != 0) + FLD_SET(r->ts_usage_flags, WT_REC_TIME_PREPARE); +} + +/* + * __rec_cell_tw_stats -- + * Gather statistics about this cell. + */ +static inline void +__rec_cell_tw_stats(WT_RECONCILE *r, WT_TIME_WINDOW *tw) +{ + if (tw->durable_start_ts != WT_TS_NONE) + ++r->count_durable_start_ts; + if (tw->start_ts != WT_TS_NONE) + ++r->count_start_ts; + if (tw->start_txn != WT_TXN_NONE) + ++r->count_start_txn; + if (tw->durable_stop_ts != WT_TS_NONE) + ++r->count_durable_stop_ts; + if (tw->stop_ts != WT_TS_MAX) + ++r->count_stop_ts; + if (tw->stop_txn != WT_TXN_MAX) + ++r->count_stop_txn; + if (tw->prepare) + ++r->count_prepare; +} + +/* + * __rec_page_time_stats_clear -- + * Clear page statistics. + */ +static inline void +__rec_page_time_stats_clear(WT_RECONCILE *r) +{ + r->count_durable_start_ts = 0; + r->count_start_ts = 0; + r->count_start_txn = 0; + r->count_durable_stop_ts = 0; + r->count_stop_ts = 0; + r->count_stop_txn = 0; + r->count_prepare = 0; + + r->ts_usage_flags = 0; +} + +/* + * __rec_page_time_stats -- + * Update statistics about this page. + */ +static inline void +__rec_page_time_stats(WT_SESSION_IMPL *session, WT_RECONCILE *r) +{ + /* Time window statistics */ + if (r->count_durable_start_ts != 0) { + WT_STAT_CONN_INCRV(session, rec_time_window_durable_start_ts, r->count_durable_start_ts); + WT_STAT_DATA_INCRV(session, rec_time_window_durable_start_ts, r->count_durable_start_ts); + WT_STAT_CONN_INCR(session, rec_time_window_pages_durable_start_ts); + WT_STAT_DATA_INCR(session, rec_time_window_pages_durable_start_ts); + } + if (r->count_start_ts != 0) { + WT_STAT_CONN_INCRV(session, rec_time_window_start_ts, r->count_start_ts); + WT_STAT_DATA_INCRV(session, rec_time_window_start_ts, r->count_start_ts); + WT_STAT_CONN_INCR(session, rec_time_window_pages_start_ts); + WT_STAT_DATA_INCR(session, rec_time_window_pages_start_ts); + } + if (r->count_start_txn != 0) { + WT_STAT_CONN_INCRV(session, rec_time_window_start_txn, r->count_start_txn); + WT_STAT_DATA_INCRV(session, rec_time_window_start_txn, r->count_start_txn); + WT_STAT_CONN_INCR(session, rec_time_window_pages_start_txn); + WT_STAT_DATA_INCR(session, rec_time_window_pages_start_txn); + } + if (r->count_durable_stop_ts != 0) { + WT_STAT_CONN_INCRV(session, rec_time_window_durable_stop_ts, r->count_durable_stop_ts); + WT_STAT_DATA_INCRV(session, rec_time_window_durable_stop_ts, r->count_durable_stop_ts); + WT_STAT_CONN_INCR(session, rec_time_window_pages_durable_stop_ts); + WT_STAT_DATA_INCR(session, rec_time_window_pages_durable_stop_ts); + } + if (r->count_stop_ts != 0) { + WT_STAT_CONN_INCRV(session, rec_time_window_stop_ts, r->count_stop_ts); + WT_STAT_DATA_INCRV(session, rec_time_window_stop_ts, r->count_stop_ts); + WT_STAT_CONN_INCR(session, rec_time_window_pages_stop_ts); + WT_STAT_DATA_INCR(session, rec_time_window_pages_stop_ts); + } + if (r->count_stop_txn != 0) { + WT_STAT_CONN_INCRV(session, rec_time_window_stop_txn, r->count_stop_txn); + WT_STAT_DATA_INCRV(session, rec_time_window_stop_txn, r->count_stop_txn); + WT_STAT_CONN_INCR(session, rec_time_window_pages_stop_txn); + WT_STAT_DATA_INCR(session, rec_time_window_pages_stop_txn); + } + + if (r->count_prepare != 0) { + WT_STAT_CONN_INCRV(session, rec_time_window_prepared, r->count_prepare); + WT_STAT_DATA_INCRV(session, rec_time_window_prepared, r->count_prepare); + WT_STAT_CONN_INCR(session, rec_time_window_pages_prepared); + WT_STAT_DATA_INCR(session, rec_time_window_pages_prepared); + } + + /* Time aggregate statistics */ + if (FLD_ISSET(r->ts_usage_flags, WT_REC_TIME_NEWEST_START_DURABLE_TS)) { + WT_STAT_CONN_INCR(session, rec_time_aggr_newest_start_durable_ts); + WT_STAT_DATA_INCR(session, rec_time_aggr_newest_start_durable_ts); + } + if (FLD_ISSET(r->ts_usage_flags, WT_REC_TIME_NEWEST_STOP_DURABLE_TS)) { + WT_STAT_CONN_INCR(session, rec_time_aggr_newest_stop_durable_ts); + WT_STAT_DATA_INCR(session, rec_time_aggr_newest_stop_durable_ts); + } + if (FLD_ISSET(r->ts_usage_flags, WT_REC_TIME_OLDEST_START_TS)) { + WT_STAT_CONN_INCR(session, rec_time_aggr_oldest_start_ts); + WT_STAT_DATA_INCR(session, rec_time_aggr_oldest_start_ts); + } + if (FLD_ISSET(r->ts_usage_flags, WT_REC_TIME_OLDEST_START_TXN)) { + WT_STAT_CONN_INCR(session, rec_time_aggr_oldest_start_txn); + WT_STAT_DATA_INCR(session, rec_time_aggr_oldest_start_txn); + } + if (FLD_ISSET(r->ts_usage_flags, WT_REC_TIME_NEWEST_STOP_TS)) { + WT_STAT_CONN_INCR(session, rec_time_aggr_newest_stop_ts); + WT_STAT_DATA_INCR(session, rec_time_aggr_newest_stop_ts); + } + if (FLD_ISSET(r->ts_usage_flags, WT_REC_TIME_NEWEST_STOP_TXN)) { + WT_STAT_CONN_INCR(session, rec_time_aggr_newest_stop_txn); + WT_STAT_DATA_INCR(session, rec_time_aggr_newest_stop_txn); + } + if (FLD_ISSET(r->ts_usage_flags, WT_REC_TIME_PREPARE)) { + WT_STAT_CONN_INCR(session, rec_time_aggr_prepared); + WT_STAT_DATA_INCR(session, rec_time_aggr_prepared); + } +} + +/* * __wt_rec_need_split -- * Check whether adding some bytes to the page requires a split. */ static inline bool __wt_rec_need_split(WT_RECONCILE *r, size_t len) { - /* - * In the case of a row-store leaf page, trigger a split if a threshold number of saved updates - * is reached. This allows pages to split for update/restore and history store eviction when - * there is no visible data causing the disk image to grow. - * - * In the case of small pages or large keys, we might try to split when a page has no updates or - * entries, which isn't possible. To consider update/restore or history store information, - * require either page entries or updates that will be attached to the image. The limit is one - * of either, but it doesn't make sense to create pages or images with few entries or updates, - * even where page sizes are small (especially as updates that will eventually become overflow - * items can throw off our calculations). Bound the combination at something reasonable. - */ - if (r->page->type == WT_PAGE_ROW_LEAF && r->entries + r->supd_next > 10) - len += r->supd_memsize; - /* Check for the disk image crossing a boundary. */ return (WT_CHECK_CROSSING_BND(r, len)); } @@ -153,6 +285,8 @@ __wt_rec_cell_build_addr(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_ADDR *add WT_ASSERT(session, addr->size != 0); } + __rec_cell_addr_stats(r, vpack == NULL ? &addr->ta : &vpack->ta); + /* * We don't check the address size because we can't store an address on an overflow page: if the * address won't fit, the overflow page's address won't fit either. This possibility must be @@ -216,8 +350,7 @@ __wt_rec_cell_build_val(WT_SESSION_IMPL *session, WT_RECONCILE *r, const void *d return (__wt_rec_cell_build_ovfl(session, r, val, WT_CELL_VALUE_OVFL, tw, rle)); } } - if (tw->prepare) - WT_STAT_DATA_INCR(session, rec_prepare_value); + __rec_cell_tw_stats(r, tw); val->cell_len = __wt_cell_pack_value(session, r, &val->cell, tw, rle, val->buf.size); val->len = val->cell_len + val->buf.size; diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index c46236b4d1d..17fb433ddbb 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -166,29 +166,29 @@ struct __wt_session_impl { #define WT_SESSION_CACHE_CURSORS 0x00000004u #define WT_SESSION_CAN_WAIT 0x00000008u #define WT_SESSION_HS_CURSOR 0x00000010u -#define WT_SESSION_IGNORE_CACHE_SIZE 0x00000020u -#define WT_SESSION_INSTANTIATE_PREPARE 0x00000040u -#define WT_SESSION_INTERNAL 0x00000080u -#define WT_SESSION_LOCKED_CHECKPOINT 0x00000100u -#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000200u -#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00000400u -#define WT_SESSION_LOCKED_HOTBACKUP_READ 0x00000800u -#define WT_SESSION_LOCKED_HOTBACKUP_WRITE 0x00001000u -#define WT_SESSION_LOCKED_METADATA 0x00002000u -#define WT_SESSION_LOCKED_PASS 0x00004000u -#define WT_SESSION_LOCKED_SCHEMA 0x00008000u -#define WT_SESSION_LOCKED_SLOT 0x00010000u -#define WT_SESSION_LOCKED_TABLE_READ 0x00020000u -#define WT_SESSION_LOCKED_TABLE_WRITE 0x00040000u -#define WT_SESSION_LOCKED_TURTLE 0x00080000u -#define WT_SESSION_LOGGING_INMEM 0x00100000u -#define WT_SESSION_NO_DATA_HANDLES 0x00200000u -#define WT_SESSION_NO_LOGGING 0x00400000u -#define WT_SESSION_NO_RECONCILE 0x00800000u -#define WT_SESSION_NO_SCHEMA_LOCK 0x01000000u -#define WT_SESSION_QUIET_CORRUPT_FILE 0x02000000u -#define WT_SESSION_READ_WONT_NEED 0x04000000u -#define WT_SESSION_RESOLVING_MODIFY 0x08000000u +#define WT_SESSION_HS_IGNORE_VISIBILITY 0x00000020u +#define WT_SESSION_IGNORE_CACHE_SIZE 0x00000040u +#define WT_SESSION_INSTANTIATE_PREPARE 0x00000080u +#define WT_SESSION_INTERNAL 0x00000100u +#define WT_SESSION_LOCKED_CHECKPOINT 0x00000200u +#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000400u +#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00000800u +#define WT_SESSION_LOCKED_HOTBACKUP_READ 0x00001000u +#define WT_SESSION_LOCKED_HOTBACKUP_WRITE 0x00002000u +#define WT_SESSION_LOCKED_METADATA 0x00004000u +#define WT_SESSION_LOCKED_PASS 0x00008000u +#define WT_SESSION_LOCKED_SCHEMA 0x00010000u +#define WT_SESSION_LOCKED_SLOT 0x00020000u +#define WT_SESSION_LOCKED_TABLE_READ 0x00040000u +#define WT_SESSION_LOCKED_TABLE_WRITE 0x00080000u +#define WT_SESSION_LOCKED_TURTLE 0x00100000u +#define WT_SESSION_LOGGING_INMEM 0x00200000u +#define WT_SESSION_NO_DATA_HANDLES 0x00400000u +#define WT_SESSION_NO_LOGGING 0x00800000u +#define WT_SESSION_NO_RECONCILE 0x01000000u +#define WT_SESSION_NO_SCHEMA_LOCK 0x02000000u +#define WT_SESSION_QUIET_CORRUPT_FILE 0x04000000u +#define WT_SESSION_READ_WONT_NEED 0x08000000u #define WT_SESSION_RESOLVING_TXN 0x10000000u #define WT_SESSION_ROLLBACK_TO_STABLE 0x20000000u #define WT_SESSION_SCHEMA_TXN 0x40000000u diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 452e1459789..8ff7015e7e6 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -614,12 +614,34 @@ struct __wt_connection_stats { int64_t perf_hist_opwrite_latency_lt10000; int64_t perf_hist_opwrite_latency_gt10000; int64_t rec_page_delete_fast; + int64_t rec_maximum_seconds; int64_t rec_pages; int64_t rec_pages_eviction; int64_t rec_pages_with_prepare; int64_t rec_pages_with_ts; int64_t rec_pages_with_txn; int64_t rec_page_delete; + int64_t rec_time_aggr_newest_start_durable_ts; + int64_t rec_time_aggr_newest_stop_durable_ts; + int64_t rec_time_aggr_newest_stop_ts; + int64_t rec_time_aggr_newest_stop_txn; + int64_t rec_time_aggr_oldest_start_ts; + int64_t rec_time_aggr_oldest_start_txn; + int64_t rec_time_aggr_prepared; + int64_t rec_time_window_pages_prepared; + int64_t rec_time_window_pages_durable_start_ts; + int64_t rec_time_window_pages_start_ts; + int64_t rec_time_window_pages_start_txn; + int64_t rec_time_window_pages_durable_stop_ts; + int64_t rec_time_window_pages_stop_ts; + int64_t rec_time_window_pages_stop_txn; + int64_t rec_time_window_prepared; + int64_t rec_time_window_durable_start_ts; + int64_t rec_time_window_start_ts; + int64_t rec_time_window_start_txn; + int64_t rec_time_window_durable_stop_ts; + int64_t rec_time_window_stop_ts; + int64_t rec_time_window_stop_txn; int64_t rec_split_stashed_bytes; int64_t rec_split_stashed_objects; int64_t session_open; @@ -860,6 +882,9 @@ struct __wt_dsrc_stats { int64_t cursor_update; int64_t cursor_update_bytes; int64_t cursor_update_bytes_changed; + int64_t hs_gc_pages_evict; + int64_t hs_gc_pages_removed; + int64_t hs_gc_pages_visited; int64_t rec_dictionary; int64_t rec_page_delete_fast; int64_t rec_suffix_compression; @@ -874,7 +899,27 @@ struct __wt_dsrc_stats { int64_t rec_pages; int64_t rec_pages_eviction; int64_t rec_page_delete; - int64_t rec_prepare_value; + int64_t rec_time_aggr_newest_start_durable_ts; + int64_t rec_time_aggr_newest_stop_durable_ts; + int64_t rec_time_aggr_newest_stop_ts; + int64_t rec_time_aggr_newest_stop_txn; + int64_t rec_time_aggr_oldest_start_ts; + int64_t rec_time_aggr_oldest_start_txn; + int64_t rec_time_aggr_prepared; + int64_t rec_time_window_pages_prepared; + int64_t rec_time_window_pages_durable_start_ts; + int64_t rec_time_window_pages_start_ts; + int64_t rec_time_window_pages_start_txn; + int64_t rec_time_window_pages_durable_stop_ts; + int64_t rec_time_window_pages_stop_ts; + int64_t rec_time_window_pages_stop_txn; + int64_t rec_time_window_prepared; + int64_t rec_time_window_durable_start_ts; + int64_t rec_time_window_start_ts; + int64_t rec_time_window_start_txn; + int64_t rec_time_window_durable_stop_ts; + int64_t rec_time_window_stop_ts; + int64_t rec_time_window_stop_txn; int64_t session_compact; int64_t txn_update_conflict; }; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 09218f3d9d1..5e6d10fd546 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -332,23 +332,22 @@ struct __wt_txn { #define WT_TXN_HAS_TS_COMMIT 0x000010u #define WT_TXN_HAS_TS_DURABLE 0x000020u #define WT_TXN_HAS_TS_PREPARE 0x000040u -#define WT_TXN_HAS_TS_READ 0x000080u -#define WT_TXN_IGNORE_PREPARE 0x000100u -#define WT_TXN_PREPARE 0x000200u -#define WT_TXN_READONLY 0x000400u -#define WT_TXN_RUNNING 0x000800u -#define WT_TXN_SHARED_TS_DURABLE 0x001000u -#define WT_TXN_SHARED_TS_READ 0x002000u -#define WT_TXN_SYNC_SET 0x004000u -#define WT_TXN_TS_COMMIT_ALWAYS 0x008000u -#define WT_TXN_TS_COMMIT_KEYS 0x010000u -#define WT_TXN_TS_COMMIT_NEVER 0x020000u -#define WT_TXN_TS_DURABLE_ALWAYS 0x040000u -#define WT_TXN_TS_DURABLE_KEYS 0x080000u -#define WT_TXN_TS_DURABLE_NEVER 0x100000u -#define WT_TXN_TS_ROUND_PREPARED 0x200000u -#define WT_TXN_TS_ROUND_READ 0x400000u -#define WT_TXN_UPDATE 0x800000u +#define WT_TXN_IGNORE_PREPARE 0x000080u +#define WT_TXN_PREPARE 0x000100u +#define WT_TXN_READONLY 0x000200u +#define WT_TXN_RUNNING 0x000400u +#define WT_TXN_SHARED_TS_DURABLE 0x000800u +#define WT_TXN_SHARED_TS_READ 0x001000u +#define WT_TXN_SYNC_SET 0x002000u +#define WT_TXN_TS_COMMIT_ALWAYS 0x004000u +#define WT_TXN_TS_COMMIT_KEYS 0x008000u +#define WT_TXN_TS_COMMIT_NEVER 0x010000u +#define WT_TXN_TS_DURABLE_ALWAYS 0x020000u +#define WT_TXN_TS_DURABLE_KEYS 0x040000u +#define WT_TXN_TS_DURABLE_NEVER 0x080000u +#define WT_TXN_TS_ROUND_PREPARED 0x100000u +#define WT_TXN_TS_ROUND_READ 0x200000u +#define WT_TXN_UPDATE 0x400000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 04389f6100a..0d00a0ce18b 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -715,7 +715,7 @@ __wt_txn_visible(WT_SESSION_IMPL *session, uint64_t id, wt_timestamp_t timestamp return (true); /* Timestamp check. */ - if (!F_ISSET(txn, WT_TXN_HAS_TS_READ) || timestamp == WT_TS_NONE) + if (!F_ISSET(txn, WT_TXN_SHARED_TS_READ) || timestamp == WT_TS_NONE) return (true); return (timestamp <= txn_shared->read_timestamp); @@ -737,7 +737,7 @@ __wt_txn_upd_visible_type(WT_SESSION_IMPL *session, WT_UPDATE *upd) if (prepare_state == WT_PREPARE_LOCKED) continue; - if (F_ISSET(session, WT_SESSION_RESOLVING_MODIFY) && upd->txnid != WT_TXN_ABORTED && + if (F_ISSET(session, WT_SESSION_HS_IGNORE_VISIBILITY) && upd->txnid != WT_TXN_ABORTED && upd->type == WT_UPDATE_STANDARD) { /* If we are resolving a modify then the btree must be the history store. */ WT_ASSERT(session, WT_IS_HS(S2BT(session))); @@ -955,7 +955,7 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint if (__wt_txn_tw_stop_visible(session, &tw) && ((!F_ISSET(&cbt->iface, WT_CURSTD_IGNORE_TOMBSTONE) && (!WT_IS_HS(S2BT(session)) || !F_ISSET(session, WT_SESSION_ROLLBACK_TO_STABLE))) || - __wt_txn_tw_stop_visible_all(session, &tw))) { + (__wt_txn_tw_stop_visible_all(session, &tw) && !WT_CURSOR_IS_DUMP(&cbt->iface)))) { cbt->upd_value->buf.data = NULL; cbt->upd_value->buf.size = 0; cbt->upd_value->tw.durable_stop_ts = tw.durable_stop_ts; @@ -975,11 +975,12 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint } /* If the start time point is visible then we need to return the ondisk value. */ - if (F_ISSET(session, WT_SESSION_RESOLVING_MODIFY) || __wt_txn_tw_start_visible(session, &tw)) { + if (F_ISSET(session, WT_SESSION_HS_IGNORE_VISIBILITY) || + __wt_txn_tw_start_visible(session, &tw)) { /* If we are resolving a modify then the btree must be the history store. */ WT_ASSERT( - session, (F_ISSET(session, WT_SESSION_RESOLVING_MODIFY) && WT_IS_HS(S2BT(session))) || - !F_ISSET(session, WT_SESSION_RESOLVING_MODIFY)); + session, (F_ISSET(session, WT_SESSION_HS_IGNORE_VISIBILITY) && WT_IS_HS(S2BT(session))) || + !F_ISSET(session, WT_SESSION_HS_IGNORE_VISIBILITY)); if (cbt->upd_value->skip_buf) { cbt->upd_value->buf.data = NULL; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 771bdb1e919..02f7deaf329 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -5615,260 +5615,334 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1317 /*! reconciliation: fast-path pages deleted */ #define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1318 +/*! reconciliation: maximum seconds spent in a reconciliation call */ +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1319 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1319 +#define WT_STAT_CONN_REC_PAGES 1320 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1320 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1321 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1321 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1322 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1322 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1323 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1323 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1324 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1324 +#define WT_STAT_CONN_REC_PAGE_DELETE 1325 +/*! + * reconciliation: pages written including an aggregated newest start + * durable timestamp + */ +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1326 +/*! + * reconciliation: pages written including an aggregated newest stop + * durable timestamp + */ +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1327 +/*! + * reconciliation: pages written including an aggregated newest stop + * timestamp + */ +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1328 +/*! + * reconciliation: pages written including an aggregated newest stop + * transaction ID + */ +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1329 +/*! + * reconciliation: pages written including an aggregated oldest start + * timestamp + */ +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1330 +/*! + * reconciliation: pages written including an aggregated oldest start + * transaction ID + */ +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TXN 1331 +/*! reconciliation: pages written including an aggregated prepare */ +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1332 +/*! reconciliation: pages written including at least one prepare state */ +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1333 +/*! + * reconciliation: pages written including at least one start durable + * timestamp + */ +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1334 +/*! reconciliation: pages written including at least one start timestamp */ +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1335 +/*! + * reconciliation: pages written including at least one start transaction + * ID + */ +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1336 +/*! + * reconciliation: pages written including at least one stop durable + * timestamp + */ +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1337 +/*! reconciliation: pages written including at least one stop timestamp */ +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1338 +/*! + * reconciliation: pages written including at least one stop transaction + * ID + */ +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1339 +/*! reconciliation: records written including a prepare state */ +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1340 +/*! reconciliation: records written including a start durable timestamp */ +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1341 +/*! reconciliation: records written including a start timestamp */ +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1342 +/*! reconciliation: records written including a start transaction ID */ +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1343 +/*! reconciliation: records written including a stop durable timestamp */ +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1344 +/*! reconciliation: records written including a stop timestamp */ +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1345 +/*! reconciliation: records written including a stop transaction ID */ +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1346 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1325 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1347 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1326 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1348 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1327 +#define WT_STAT_CONN_SESSION_OPEN 1349 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1328 +#define WT_STAT_CONN_SESSION_QUERY_TS 1350 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1329 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1351 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1330 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1352 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1331 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1353 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1332 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1354 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1333 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1355 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1334 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1356 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1335 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1357 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1336 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1358 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1337 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1359 /*! session: table import failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1338 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_FAIL 1360 /*! session: table import successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1339 +#define WT_STAT_CONN_SESSION_TABLE_IMPORT_SUCCESS 1361 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1340 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1362 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1341 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1363 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1342 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1364 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1343 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1365 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1344 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1366 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1345 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1367 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1346 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1368 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1347 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1369 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1348 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1370 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1349 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1371 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1350 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1372 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1351 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1373 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1352 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1374 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1353 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1375 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1354 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1376 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1355 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1377 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1356 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1378 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1357 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1379 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1358 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1380 /*! thread-yield: log server sync yielded for log write */ -#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1359 +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1381 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1360 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1382 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1361 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1383 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1362 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1384 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1363 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1385 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1364 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1386 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1365 +#define WT_STAT_CONN_PAGE_SLEEP 1387 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1366 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1388 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1367 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1389 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1368 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COUNT 1390 /*! transaction: durable timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1369 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_WALKED 1391 /*! transaction: durable timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1370 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_EMPTY 1392 /*! transaction: durable timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1371 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_HEAD 1393 /*! transaction: durable timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1372 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_INSERTS 1394 /*! transaction: durable timestamp queue length */ -#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1373 +#define WT_STAT_CONN_TXN_DURABLE_QUEUE_LEN 1395 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1374 +#define WT_STAT_CONN_TXN_PREPARE 1396 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1375 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1397 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1376 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1398 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1377 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1399 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1378 +#define WT_STAT_CONN_TXN_QUERY_TS 1400 /*! transaction: read timestamp queue entries walked */ -#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1379 +#define WT_STAT_CONN_TXN_READ_QUEUE_WALKED 1401 /*! transaction: read timestamp queue insert to empty */ -#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1380 +#define WT_STAT_CONN_TXN_READ_QUEUE_EMPTY 1402 /*! transaction: read timestamp queue inserts to head */ -#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1381 +#define WT_STAT_CONN_TXN_READ_QUEUE_HEAD 1403 /*! transaction: read timestamp queue inserts total */ -#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1382 +#define WT_STAT_CONN_TXN_READ_QUEUE_INSERTS 1404 /*! transaction: read timestamp queue length */ -#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1383 +#define WT_STAT_CONN_TXN_READ_QUEUE_LEN 1405 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1384 +#define WT_STAT_CONN_TXN_RTS 1406 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1385 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1407 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1386 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1408 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1387 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1409 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1388 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1410 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1389 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1411 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1390 +#define WT_STAT_CONN_TXN_SET_TS 1412 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1391 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1413 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1392 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1414 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1393 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1415 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1394 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1416 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1395 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1417 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1396 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1418 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1397 +#define WT_STAT_CONN_TXN_BEGIN 1419 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1398 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1420 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1399 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1421 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1400 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1422 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1401 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1423 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1402 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1424 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1403 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1425 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1404 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1426 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1405 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1427 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1406 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1428 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1407 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1429 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1408 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1430 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1409 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1431 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1410 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1432 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1411 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1433 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1412 +#define WT_STAT_CONN_TXN_CHECKPOINT 1434 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1413 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1435 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1414 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1436 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1415 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1437 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1416 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1438 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1417 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1439 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1418 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1440 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1419 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1441 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1420 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1442 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1421 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1443 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1422 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1444 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1423 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1445 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1424 +#define WT_STAT_CONN_TXN_SYNC 1446 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1425 +#define WT_STAT_CONN_TXN_COMMIT 1447 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1426 +#define WT_STAT_CONN_TXN_ROLLBACK 1448 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1427 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1449 /*! * @} @@ -6248,43 +6322,119 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2128 /*! cursor: update value size change */ #define WT_STAT_DSRC_CURSOR_UPDATE_BYTES_CHANGED 2129 +/*! history: history pages added for eviction during garbage collection */ +#define WT_STAT_DSRC_HS_GC_PAGES_EVICT 2130 +/*! history: history pages removed for garbage collection */ +#define WT_STAT_DSRC_HS_GC_PAGES_REMOVED 2131 +/*! history: history pages visited for garbage collection */ +#define WT_STAT_DSRC_HS_GC_PAGES_VISITED 2132 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2130 +#define WT_STAT_DSRC_REC_DICTIONARY 2133 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2131 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2134 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2132 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2135 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2133 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2136 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2134 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2137 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2135 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2138 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2136 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2139 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2137 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2140 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2138 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2141 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2139 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2142 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2140 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2143 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2141 +#define WT_STAT_DSRC_REC_PAGES 2144 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2142 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2145 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2143 -/*! reconciliation: prepared values written */ -#define WT_STAT_DSRC_REC_PREPARE_VALUE 2144 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2146 +/*! + * reconciliation: pages written including an aggregated newest start + * durable timestamp + */ +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2147 +/*! + * reconciliation: pages written including an aggregated newest stop + * durable timestamp + */ +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2148 +/*! + * reconciliation: pages written including an aggregated newest stop + * timestamp + */ +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2149 +/*! + * reconciliation: pages written including an aggregated newest stop + * transaction ID + */ +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2150 +/*! + * reconciliation: pages written including an aggregated oldest start + * timestamp + */ +#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2151 +/*! + * reconciliation: pages written including an aggregated oldest start + * transaction ID + */ +#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TXN 2152 +/*! reconciliation: pages written including an aggregated prepare */ +#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2153 +/*! reconciliation: pages written including at least one prepare */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_PREPARED 2154 +/*! + * reconciliation: pages written including at least one start durable + * timestamp + */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2155 +/*! reconciliation: pages written including at least one start timestamp */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TS 2156 +/*! + * reconciliation: pages written including at least one start transaction + * ID + */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2157 +/*! + * reconciliation: pages written including at least one stop durable + * timestamp + */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2158 +/*! reconciliation: pages written including at least one stop timestamp */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2159 +/*! + * reconciliation: pages written including at least one stop transaction + * ID + */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2160 +/*! reconciliation: records written including a prepare */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_PREPARED 2161 +/*! reconciliation: records written including a start durable timestamp */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2162 +/*! reconciliation: records written including a start timestamp */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2163 +/*! reconciliation: records written including a start transaction ID */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2164 +/*! reconciliation: records written including a stop durable timestamp */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2165 +/*! reconciliation: records written including a stop timestamp */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2166 +/*! reconciliation: records written including a stop transaction ID */ +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2167 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2145 +#define WT_STAT_DSRC_SESSION_COMPACT 2168 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2146 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2169 /*! * @} diff --git a/src/third_party/wiredtiger/src/reconcile/rec_row.c b/src/third_party/wiredtiger/src/reconcile/rec_row.c index 687750b7faf..9ee0a8987e7 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_row.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_row.c @@ -205,8 +205,11 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) val = &r->v; WT_RET(__rec_cell_build_leaf_key(session, r, /* Build key cell */ cursor->key.data, cursor->key.size, &ovfl_key)); - WT_RET(__wt_rec_cell_build_val(session, r, cursor->value.data, /* Build value cell */ - cursor->value.size, &tw, 0)); + if (cursor->value.size == 0) + val->len = 0; + else + WT_RET(__wt_rec_cell_build_val(session, r, cursor->value.data, /* Build value cell */ + cursor->value.size, &tw, 0)); /* Boundary: split or write the page. */ if (WT_CROSSING_SPLIT_BND(r, key->len + val->len)) { @@ -582,8 +585,11 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) session, r, cbt->iface.value.data, cbt->iface.value.size, &tw, 0)); break; case WT_UPDATE_STANDARD: - /* Take the value from the update. */ - WT_RET(__wt_rec_cell_build_val(session, r, upd->data, upd->size, &tw, 0)); + if (upd->size == 0 && WT_TIME_WINDOW_IS_EMPTY(&tw)) + val->len = 0; + else + /* Take the value from the update. */ + WT_RET(__wt_rec_cell_build_val(session, r, upd->data, upd->size, &tw, 0)); break; case WT_UPDATE_TOMBSTONE: continue; diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 9bfcbefd848..d083f076c69 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -31,8 +31,10 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage { WT_DECL_RET; WT_PAGE *page; + uint64_t start, now; bool no_reconcile_set, page_locked; + __wt_seconds(session, &start); page = ref->page; __wt_verbose(session, WT_VERB_RECONCILE, "%p reconcile %s (%s%s)", (void *)ref, @@ -91,6 +93,12 @@ err: WT_PAGE_UNLOCK(session, page); if (!no_reconcile_set) F_CLR(session, WT_SESSION_NO_RECONCILE); + + /* Track the longest reconciliation, ignoring races (it's just a statistic). */ + __wt_seconds(session, &now); + if (now - start > S2C(session)->rec_maximum_seconds) + S2C(session)->rec_maximum_seconds = now - start; + return (ret); } @@ -1817,6 +1825,9 @@ __rec_split_write(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REC_CHUNK *chunk __rec_compression_adjust( session, btree->maxleafpage, compressed_size, last_block, &btree->maxleafpage_precomp); + /* Update the per-page reconciliation time statistics now that we've written something. */ + __rec_page_time_stats(session, r); + copy_image: #ifdef HAVE_DIAGNOSTIC /* @@ -1834,6 +1845,9 @@ copy_image: if (F_ISSET(r, WT_REC_SCRUB) || multi->supd_restore) WT_RET(__wt_memdup(session, chunk->image.data, chunk->image.size, &multi->disk_image)); + /* Whether we wrote or not, clear the accumulated time statistics. */ + __rec_page_time_stats_clear(r); + return (0); } diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 7be8e90982f..c3588bffad3 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -81,7 +81,10 @@ static const char *const __stats_dsrc_desc[] = { "cursor: remove key bytes removed", "cursor: reserve calls", "cursor: reset calls", "cursor: search calls", "cursor: search near calls", "cursor: truncate calls", "cursor: update calls", "cursor: update key and value bytes", "cursor: update value size change", - "reconciliation: dictionary matches", "reconciliation: fast-path pages deleted", + "history: history pages added for eviction during garbage collection", + "history: history pages removed for garbage collection", + "history: history pages visited for garbage collection", "reconciliation: dictionary matches", + "reconciliation: fast-path pages deleted", "reconciliation: internal page key bytes discarded using suffix compression", "reconciliation: internal page multi-block writes", "reconciliation: internal-page overflow keys", "reconciliation: leaf page key bytes discarded using prefix compression", @@ -89,7 +92,27 @@ static const char *const __stats_dsrc_desc[] = { "reconciliation: maximum blocks required for a page", "reconciliation: overflow values written", "reconciliation: page checksum matches", "reconciliation: page reconciliation calls", "reconciliation: page reconciliation calls for eviction", "reconciliation: pages deleted", - "reconciliation: prepared values written", "session: object compaction", + "reconciliation: pages written including an aggregated newest start durable timestamp ", + "reconciliation: pages written including an aggregated newest stop durable timestamp ", + "reconciliation: pages written including an aggregated newest stop timestamp ", + "reconciliation: pages written including an aggregated newest stop transaction ID", + "reconciliation: pages written including an aggregated oldest start timestamp ", + "reconciliation: pages written including an aggregated oldest start transaction ID ", + "reconciliation: pages written including an aggregated prepare", + "reconciliation: pages written including at least one prepare", + "reconciliation: pages written including at least one start durable timestamp", + "reconciliation: pages written including at least one start timestamp", + "reconciliation: pages written including at least one start transaction ID", + "reconciliation: pages written including at least one stop durable timestamp", + "reconciliation: pages written including at least one stop timestamp", + "reconciliation: pages written including at least one stop transaction ID", + "reconciliation: records written including a prepare", + "reconciliation: records written including a start durable timestamp", + "reconciliation: records written including a start timestamp", + "reconciliation: records written including a start transaction ID", + "reconciliation: records written including a stop durable timestamp", + "reconciliation: records written including a stop timestamp", + "reconciliation: records written including a stop transaction ID", "session: object compaction", "transaction: update conflicts", }; @@ -261,6 +284,9 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cursor_update = 0; stats->cursor_update_bytes = 0; stats->cursor_update_bytes_changed = 0; + stats->hs_gc_pages_evict = 0; + stats->hs_gc_pages_removed = 0; + stats->hs_gc_pages_visited = 0; stats->rec_dictionary = 0; stats->rec_page_delete_fast = 0; stats->rec_suffix_compression = 0; @@ -275,7 +301,27 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->rec_pages = 0; stats->rec_pages_eviction = 0; stats->rec_page_delete = 0; - stats->rec_prepare_value = 0; + stats->rec_time_aggr_newest_start_durable_ts = 0; + stats->rec_time_aggr_newest_stop_durable_ts = 0; + stats->rec_time_aggr_newest_stop_ts = 0; + stats->rec_time_aggr_newest_stop_txn = 0; + stats->rec_time_aggr_oldest_start_ts = 0; + stats->rec_time_aggr_oldest_start_txn = 0; + stats->rec_time_aggr_prepared = 0; + stats->rec_time_window_pages_prepared = 0; + stats->rec_time_window_pages_durable_start_ts = 0; + stats->rec_time_window_pages_start_ts = 0; + stats->rec_time_window_pages_start_txn = 0; + stats->rec_time_window_pages_durable_stop_ts = 0; + stats->rec_time_window_pages_stop_ts = 0; + stats->rec_time_window_pages_stop_txn = 0; + stats->rec_time_window_prepared = 0; + stats->rec_time_window_durable_start_ts = 0; + stats->rec_time_window_start_ts = 0; + stats->rec_time_window_start_txn = 0; + stats->rec_time_window_durable_stop_ts = 0; + stats->rec_time_window_stop_ts = 0; + stats->rec_time_window_stop_txn = 0; stats->session_compact = 0; stats->txn_update_conflict = 0; } @@ -434,6 +480,9 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->cursor_update += from->cursor_update; to->cursor_update_bytes += from->cursor_update_bytes; to->cursor_update_bytes_changed += from->cursor_update_bytes_changed; + to->hs_gc_pages_evict += from->hs_gc_pages_evict; + to->hs_gc_pages_removed += from->hs_gc_pages_removed; + to->hs_gc_pages_visited += from->hs_gc_pages_visited; to->rec_dictionary += from->rec_dictionary; to->rec_page_delete_fast += from->rec_page_delete_fast; to->rec_suffix_compression += from->rec_suffix_compression; @@ -449,7 +498,27 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->rec_pages += from->rec_pages; to->rec_pages_eviction += from->rec_pages_eviction; to->rec_page_delete += from->rec_page_delete; - to->rec_prepare_value += from->rec_prepare_value; + to->rec_time_aggr_newest_start_durable_ts += from->rec_time_aggr_newest_start_durable_ts; + to->rec_time_aggr_newest_stop_durable_ts += from->rec_time_aggr_newest_stop_durable_ts; + to->rec_time_aggr_newest_stop_ts += from->rec_time_aggr_newest_stop_ts; + to->rec_time_aggr_newest_stop_txn += from->rec_time_aggr_newest_stop_txn; + to->rec_time_aggr_oldest_start_ts += from->rec_time_aggr_oldest_start_ts; + to->rec_time_aggr_oldest_start_txn += from->rec_time_aggr_oldest_start_txn; + to->rec_time_aggr_prepared += from->rec_time_aggr_prepared; + to->rec_time_window_pages_prepared += from->rec_time_window_pages_prepared; + to->rec_time_window_pages_durable_start_ts += from->rec_time_window_pages_durable_start_ts; + to->rec_time_window_pages_start_ts += from->rec_time_window_pages_start_ts; + to->rec_time_window_pages_start_txn += from->rec_time_window_pages_start_txn; + to->rec_time_window_pages_durable_stop_ts += from->rec_time_window_pages_durable_stop_ts; + to->rec_time_window_pages_stop_ts += from->rec_time_window_pages_stop_ts; + to->rec_time_window_pages_stop_txn += from->rec_time_window_pages_stop_txn; + to->rec_time_window_prepared += from->rec_time_window_prepared; + to->rec_time_window_durable_start_ts += from->rec_time_window_durable_start_ts; + to->rec_time_window_start_ts += from->rec_time_window_start_ts; + to->rec_time_window_start_txn += from->rec_time_window_start_txn; + to->rec_time_window_durable_stop_ts += from->rec_time_window_durable_stop_ts; + to->rec_time_window_stop_ts += from->rec_time_window_stop_ts; + to->rec_time_window_stop_txn += from->rec_time_window_stop_txn; to->session_compact += from->session_compact; to->txn_update_conflict += from->txn_update_conflict; } @@ -605,6 +674,9 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->cursor_update += WT_STAT_READ(from, cursor_update); to->cursor_update_bytes += WT_STAT_READ(from, cursor_update_bytes); to->cursor_update_bytes_changed += WT_STAT_READ(from, cursor_update_bytes_changed); + to->hs_gc_pages_evict += WT_STAT_READ(from, hs_gc_pages_evict); + to->hs_gc_pages_removed += WT_STAT_READ(from, hs_gc_pages_removed); + to->hs_gc_pages_visited += WT_STAT_READ(from, hs_gc_pages_visited); to->rec_dictionary += WT_STAT_READ(from, rec_dictionary); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_suffix_compression += WT_STAT_READ(from, rec_suffix_compression); @@ -620,7 +692,31 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->rec_pages += WT_STAT_READ(from, rec_pages); to->rec_pages_eviction += WT_STAT_READ(from, rec_pages_eviction); to->rec_page_delete += WT_STAT_READ(from, rec_page_delete); - to->rec_prepare_value += WT_STAT_READ(from, rec_prepare_value); + to->rec_time_aggr_newest_start_durable_ts += + WT_STAT_READ(from, rec_time_aggr_newest_start_durable_ts); + to->rec_time_aggr_newest_stop_durable_ts += + WT_STAT_READ(from, rec_time_aggr_newest_stop_durable_ts); + to->rec_time_aggr_newest_stop_ts += WT_STAT_READ(from, rec_time_aggr_newest_stop_ts); + to->rec_time_aggr_newest_stop_txn += WT_STAT_READ(from, rec_time_aggr_newest_stop_txn); + to->rec_time_aggr_oldest_start_ts += WT_STAT_READ(from, rec_time_aggr_oldest_start_ts); + to->rec_time_aggr_oldest_start_txn += WT_STAT_READ(from, rec_time_aggr_oldest_start_txn); + to->rec_time_aggr_prepared += WT_STAT_READ(from, rec_time_aggr_prepared); + to->rec_time_window_pages_prepared += WT_STAT_READ(from, rec_time_window_pages_prepared); + to->rec_time_window_pages_durable_start_ts += + WT_STAT_READ(from, rec_time_window_pages_durable_start_ts); + to->rec_time_window_pages_start_ts += WT_STAT_READ(from, rec_time_window_pages_start_ts); + to->rec_time_window_pages_start_txn += WT_STAT_READ(from, rec_time_window_pages_start_txn); + to->rec_time_window_pages_durable_stop_ts += + WT_STAT_READ(from, rec_time_window_pages_durable_stop_ts); + to->rec_time_window_pages_stop_ts += WT_STAT_READ(from, rec_time_window_pages_stop_ts); + to->rec_time_window_pages_stop_txn += WT_STAT_READ(from, rec_time_window_pages_stop_txn); + to->rec_time_window_prepared += WT_STAT_READ(from, rec_time_window_prepared); + to->rec_time_window_durable_start_ts += WT_STAT_READ(from, rec_time_window_durable_start_ts); + to->rec_time_window_start_ts += WT_STAT_READ(from, rec_time_window_start_ts); + to->rec_time_window_start_txn += WT_STAT_READ(from, rec_time_window_start_txn); + to->rec_time_window_durable_stop_ts += WT_STAT_READ(from, rec_time_window_durable_stop_ts); + to->rec_time_window_stop_ts += WT_STAT_READ(from, rec_time_window_stop_ts); + to->rec_time_window_stop_txn += WT_STAT_READ(from, rec_time_window_stop_txn); to->session_compact += WT_STAT_READ(from, session_compact); to->txn_update_conflict += WT_STAT_READ(from, txn_update_conflict); } @@ -844,13 +940,37 @@ static const char *const __stats_connection_desc[] = { "perf: operation write latency histogram (bucket 3) - 500-999us", "perf: operation write latency histogram (bucket 4) - 1000-9999us", "perf: operation write latency histogram (bucket 5) - 10000us+", - "reconciliation: fast-path pages deleted", "reconciliation: page reconciliation calls", + "reconciliation: fast-path pages deleted", + "reconciliation: maximum seconds spent in a reconciliation call", + "reconciliation: page reconciliation calls", "reconciliation: page reconciliation calls for eviction", "reconciliation: page reconciliation calls that resulted in values with prepared transaction " "metadata", "reconciliation: page reconciliation calls that resulted in values with timestamps", "reconciliation: page reconciliation calls that resulted in values with transaction ids", - "reconciliation: pages deleted", "reconciliation: split bytes currently awaiting free", + "reconciliation: pages deleted", + "reconciliation: pages written including an aggregated newest start durable timestamp ", + "reconciliation: pages written including an aggregated newest stop durable timestamp ", + "reconciliation: pages written including an aggregated newest stop timestamp ", + "reconciliation: pages written including an aggregated newest stop transaction ID", + "reconciliation: pages written including an aggregated oldest start timestamp ", + "reconciliation: pages written including an aggregated oldest start transaction ID ", + "reconciliation: pages written including an aggregated prepare", + "reconciliation: pages written including at least one prepare state", + "reconciliation: pages written including at least one start durable timestamp", + "reconciliation: pages written including at least one start timestamp", + "reconciliation: pages written including at least one start transaction ID", + "reconciliation: pages written including at least one stop durable timestamp", + "reconciliation: pages written including at least one stop timestamp", + "reconciliation: pages written including at least one stop transaction ID", + "reconciliation: records written including a prepare state", + "reconciliation: records written including a start durable timestamp", + "reconciliation: records written including a start timestamp", + "reconciliation: records written including a start transaction ID", + "reconciliation: records written including a stop durable timestamp", + "reconciliation: records written including a stop timestamp", + "reconciliation: records written including a stop transaction ID", + "reconciliation: split bytes currently awaiting free", "reconciliation: split objects currently awaiting free", "session: open session count", "session: session query timestamp calls", "session: table alter failed calls", "session: table alter successful calls", "session: table alter unchanged and skipped", @@ -1285,12 +1405,34 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->perf_hist_opwrite_latency_lt10000 = 0; stats->perf_hist_opwrite_latency_gt10000 = 0; stats->rec_page_delete_fast = 0; + /* not clearing rec_maximum_seconds */ stats->rec_pages = 0; stats->rec_pages_eviction = 0; stats->rec_pages_with_prepare = 0; stats->rec_pages_with_ts = 0; stats->rec_pages_with_txn = 0; stats->rec_page_delete = 0; + stats->rec_time_aggr_newest_start_durable_ts = 0; + stats->rec_time_aggr_newest_stop_durable_ts = 0; + stats->rec_time_aggr_newest_stop_ts = 0; + stats->rec_time_aggr_newest_stop_txn = 0; + stats->rec_time_aggr_oldest_start_ts = 0; + stats->rec_time_aggr_oldest_start_txn = 0; + stats->rec_time_aggr_prepared = 0; + stats->rec_time_window_pages_prepared = 0; + stats->rec_time_window_pages_durable_start_ts = 0; + stats->rec_time_window_pages_start_ts = 0; + stats->rec_time_window_pages_start_txn = 0; + stats->rec_time_window_pages_durable_stop_ts = 0; + stats->rec_time_window_pages_stop_ts = 0; + stats->rec_time_window_pages_stop_txn = 0; + stats->rec_time_window_prepared = 0; + stats->rec_time_window_durable_start_ts = 0; + stats->rec_time_window_start_ts = 0; + stats->rec_time_window_start_txn = 0; + stats->rec_time_window_durable_stop_ts = 0; + stats->rec_time_window_stop_ts = 0; + stats->rec_time_window_stop_txn = 0; /* not clearing rec_split_stashed_bytes */ /* not clearing rec_split_stashed_objects */ /* not clearing session_open */ @@ -1752,12 +1894,38 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->perf_hist_opwrite_latency_lt10000 += WT_STAT_READ(from, perf_hist_opwrite_latency_lt10000); to->perf_hist_opwrite_latency_gt10000 += WT_STAT_READ(from, perf_hist_opwrite_latency_gt10000); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); + to->rec_maximum_seconds += WT_STAT_READ(from, rec_maximum_seconds); to->rec_pages += WT_STAT_READ(from, rec_pages); to->rec_pages_eviction += WT_STAT_READ(from, rec_pages_eviction); to->rec_pages_with_prepare += WT_STAT_READ(from, rec_pages_with_prepare); to->rec_pages_with_ts += WT_STAT_READ(from, rec_pages_with_ts); to->rec_pages_with_txn += WT_STAT_READ(from, rec_pages_with_txn); to->rec_page_delete += WT_STAT_READ(from, rec_page_delete); + to->rec_time_aggr_newest_start_durable_ts += + WT_STAT_READ(from, rec_time_aggr_newest_start_durable_ts); + to->rec_time_aggr_newest_stop_durable_ts += + WT_STAT_READ(from, rec_time_aggr_newest_stop_durable_ts); + to->rec_time_aggr_newest_stop_ts += WT_STAT_READ(from, rec_time_aggr_newest_stop_ts); + to->rec_time_aggr_newest_stop_txn += WT_STAT_READ(from, rec_time_aggr_newest_stop_txn); + to->rec_time_aggr_oldest_start_ts += WT_STAT_READ(from, rec_time_aggr_oldest_start_ts); + to->rec_time_aggr_oldest_start_txn += WT_STAT_READ(from, rec_time_aggr_oldest_start_txn); + to->rec_time_aggr_prepared += WT_STAT_READ(from, rec_time_aggr_prepared); + to->rec_time_window_pages_prepared += WT_STAT_READ(from, rec_time_window_pages_prepared); + to->rec_time_window_pages_durable_start_ts += + WT_STAT_READ(from, rec_time_window_pages_durable_start_ts); + to->rec_time_window_pages_start_ts += WT_STAT_READ(from, rec_time_window_pages_start_ts); + to->rec_time_window_pages_start_txn += WT_STAT_READ(from, rec_time_window_pages_start_txn); + to->rec_time_window_pages_durable_stop_ts += + WT_STAT_READ(from, rec_time_window_pages_durable_stop_ts); + to->rec_time_window_pages_stop_ts += WT_STAT_READ(from, rec_time_window_pages_stop_ts); + to->rec_time_window_pages_stop_txn += WT_STAT_READ(from, rec_time_window_pages_stop_txn); + to->rec_time_window_prepared += WT_STAT_READ(from, rec_time_window_prepared); + to->rec_time_window_durable_start_ts += WT_STAT_READ(from, rec_time_window_durable_start_ts); + to->rec_time_window_start_ts += WT_STAT_READ(from, rec_time_window_start_ts); + to->rec_time_window_start_txn += WT_STAT_READ(from, rec_time_window_start_txn); + to->rec_time_window_durable_stop_ts += WT_STAT_READ(from, rec_time_window_durable_stop_ts); + to->rec_time_window_stop_ts += WT_STAT_READ(from, rec_time_window_stop_ts); + to->rec_time_window_stop_txn += WT_STAT_READ(from, rec_time_window_stop_txn); to->rec_split_stashed_bytes += WT_STAT_READ(from, rec_split_stashed_bytes); to->rec_split_stashed_objects += WT_STAT_READ(from, rec_split_stashed_objects); to->session_open += WT_STAT_READ(from, session_open); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index dac020c35f1..af9aac63b11 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -889,14 +889,15 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, * Aborted updates can exist in the update chain of our transaction. Generally this will occur * due to a reserved update. As such we should skip over these updates. */ - for (; upd->txnid == WT_TXN_ABORTED; upd = upd->next) + for (; upd != NULL && upd->txnid == WT_TXN_ABORTED; upd = upd->next) ; /* * The head of the update chain is not a prepared update, which means all the prepared updates - * of the key are resolved. + * of the key are resolved. The head of the update chain can also be null in the scenario that + * we rolled back all associated updates in the previous iteration of this function. */ - if (upd->prepare_state != WT_PREPARE_INPROGRESS) + if (upd == NULL || upd->prepare_state != WT_PREPARE_INPROGRESS) return (0); /* diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 4d09899ddf5..eb3ab25b823 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -605,8 +605,8 @@ __checkpoint_prepare(WT_SESSION_IMPL *session, bool *trackingp, const char *cfg[ * We rely on having the global transaction data locked so the oldest timestamp can't move past * the stable timestamp. */ - WT_ASSERT(session, !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT | WT_TXN_HAS_TS_READ | - WT_TXN_SHARED_TS_DURABLE | WT_TXN_SHARED_TS_READ)); + WT_ASSERT(session, + !F_ISSET(txn, WT_TXN_HAS_TS_COMMIT | WT_TXN_SHARED_TS_DURABLE | WT_TXN_SHARED_TS_READ)); if (use_timestamp) { /* @@ -773,8 +773,9 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) logging = FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED); - /* Reset the maximum page size seen by eviction. */ - conn->cache->evict_max_page_size = 0; + /* Reset the statistics tracked per checkpoint. */ + cache->evict_max_page_size = 0; + conn->rec_maximum_seconds = 0; /* Initialize the verbose tracking timer */ __wt_epoch(session, &conn->ckpt_timer_start); @@ -1702,20 +1703,20 @@ __checkpoint_tree_helper(WT_SESSION_IMPL *session, const char *cfg[]) txn = session->txn; /* Are we using a read timestamp for this checkpoint transaction? */ - with_timestamp = F_ISSET(txn, WT_TXN_HAS_TS_READ); + with_timestamp = F_ISSET(txn, WT_TXN_SHARED_TS_READ); /* * For tables with immediate durability (indicated by having logging enabled), ignore any read * timestamp configured for the checkpoint. */ if (__wt_btree_immediately_durable(session)) - F_CLR(txn, WT_TXN_HAS_TS_READ); + F_CLR(txn, WT_TXN_SHARED_TS_READ); ret = __checkpoint_tree(session, true, cfg); /* Restore the use of the timestamp for other tables. */ if (with_timestamp) - F_SET(txn, WT_TXN_HAS_TS_READ); + F_SET(txn, WT_TXN_SHARED_TS_READ); /* * Whatever happened, we aren't visiting this tree again in this checkpoint. Don't keep updates diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 37a91dd0d6a..e52cc139906 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -428,7 +428,7 @@ __wt_txn_ts_log(WT_SESSION_IMPL *session) durable = txn->durable_timestamp; if (F_ISSET(txn, WT_TXN_HAS_TS_PREPARE)) prepare = txn->prepare_timestamp; - if (F_ISSET(txn, WT_TXN_HAS_TS_READ)) + if (F_ISSET(txn, WT_TXN_SHARED_TS_READ)) read = txn_shared->read_timestamp; __wt_epoch(session, &t); diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index a2a3dea7a5c..ba311cc93f0 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -830,7 +830,7 @@ __wt_txn_set_read_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t read_ts) " isolation"); /* Read timestamps can't change once set. */ - if (F_ISSET(txn, WT_TXN_HAS_TS_READ)) + if (F_ISSET(txn, WT_TXN_SHARED_TS_READ)) WT_RET_MSG(session, EINVAL, "a read_timestamp" " may only be set once per transaction"); @@ -1152,7 +1152,7 @@ __wt_txn_publish_read_timestamp(WT_SESSION_IMPL *session) ++txn_global->read_timestampq_len; WT_STAT_CONN_INCR(session, txn_read_queue_inserts); txn_shared->clear_read_q = false; - F_SET(txn, WT_TXN_HAS_TS_READ | WT_TXN_SHARED_TS_READ); + F_SET(txn, WT_TXN_SHARED_TS_READ); __wt_writeunlock(session, &txn_global->read_timestamp_rwlock); } diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index 9ea4e10e7aa..44897ddf044 100755 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -51,6 +51,7 @@ int util_dump(WT_SESSION *session, int argc, char *argv[]) { WT_CURSOR *cursor; + WT_CURSOR_DUMP *hs_dump_cursor; WT_DECL_ITEM(tmp); WT_DECL_RET; WT_SESSION_IMPL *session_impl; @@ -61,6 +62,7 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) session_impl = (WT_SESSION_IMPL *)session; cursor = NULL; + hs_dump_cursor = NULL; checkpoint = ofile = simpleuri = uri = timestamp = NULL; hex = json = reverse = false; while ((ch = __wt_getopt(progname, argc, argv, "c:f:t:jrx")) != EOF) @@ -154,8 +156,11 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) * nothing will be visible. The only exception is if we've supplied a timestamp in which * case, we're specifically interested in what is visible at a given read timestamp. */ - if (WT_STREQ(simpleuri, WT_HS_URI) && timestamp == NULL) - F_SET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); + if (WT_STREQ(simpleuri, WT_HS_URI) && timestamp == NULL) { + hs_dump_cursor = (WT_CURSOR_DUMP *)cursor; + /* Set the "ignore tombstone" flag on the underlying cursor. */ + F_SET(hs_dump_cursor->child, WT_CURSTD_IGNORE_TOMBSTONE); + } if (dump_config(session, simpleuri, cursor, hex, json) != 0) goto err; @@ -164,9 +169,11 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) if (json && dump_json_table_end(session) != 0) goto err; - F_CLR(cursor, WT_CURSTD_IGNORE_TOMBSTONE); + if (hs_dump_cursor != NULL) + F_CLR(hs_dump_cursor->child, WT_CURSTD_IGNORE_TOMBSTONE); ret = cursor->close(cursor); cursor = NULL; + hs_dump_cursor = NULL; if (ret != 0) { (void)util_err(session, ret, NULL); goto err; @@ -181,7 +188,8 @@ err: } if (cursor != NULL) { - F_CLR(cursor, WT_CURSTD_IGNORE_TOMBSTONE); + if (hs_dump_cursor != NULL) + F_CLR(hs_dump_cursor->child, WT_CURSTD_IGNORE_TOMBSTONE); if ((ret = cursor->close(cursor)) != 0) ret = util_err(session, ret, NULL); } diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 36d6a66b785..bdabad90184 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -2178,7 +2178,7 @@ tasks: posix_configure_flags: --enable-diagnostic --with-builtins=lz4,snappy,zlib - func: "format test script" vars: - test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=`ls /usr/bin/llvm-symbolizer* | tail -1` + test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer # run for 2 hours ( 2 * 60 = 120 minutes), don't stop at failed tests, use default config format_test_script_args: -t 120 @@ -2196,7 +2196,7 @@ tasks: # to emulate the original Jenkins job's test coverage, we are running the smoke test 16 times # run smoke tests, don't stop at failed tests, use default config vars: - test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=`ls /usr/bin/llvm-symbolizer* | tail -1` + test_env_vars: ASAN_OPTIONS="detect_leaks=1:abort_on_error=1:disable_coredump=0" ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer format_test_script_args: -S times: 16 diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 6ce8ba12ca7..b16835c7eec 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -45,7 +45,7 @@ check_copy(void) path = dmalloc(len); testutil_check(__wt_snprintf(path, len, "%s/BACKUP", g.home)); - wts_open(path, false, &conn); + wts_open(path, false, &conn, true); testutil_checkfmt(conn->open_session(conn, NULL, NULL, &session), "%s", path); diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 6bc213a65ef..9f864aeec32 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -397,7 +397,7 @@ void wts_create(void); void wts_dump(const char *, bool); void wts_init(void); void wts_load(void); -void wts_open(const char *, bool, WT_CONNECTION **); +void wts_open(const char *, bool, WT_CONNECTION **, bool); void wts_read_scan(void); void wts_rebalance(void); void wts_reopen(void); diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index b9aca837215..8d1a2e7ed0f 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -58,7 +58,8 @@ corrupt(void) wt_off_t offset; size_t len, nw; int fd, ret; - char buf[8 * 1024], copycmd[2 * 1024]; + char copycmd[2 * 1024], path[1024]; + const char *smash; /* * If it's a single Btree file (not LSM), open the file, and corrupt roughly 2% of the file at a @@ -67,14 +68,14 @@ corrupt(void) * It's a little tricky: if the data source is a file, we're looking for "wt", if the data * source is a table, we're looking for "wt.wt". */ - testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s", g.home, WT_NAME)); - if ((fd = open(buf, O_RDWR)) != -1) { + testutil_check(__wt_snprintf(path, sizeof(path), "%s/%s", g.home, WT_NAME)); + if ((fd = open(path, O_RDWR)) != -1) { testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s %s/SALVAGE.copy/%s.corrupted", g.home, WT_NAME, g.home, WT_NAME)); goto found; } - testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME)); - if ((fd = open(buf, O_RDWR)) != -1) { + testutil_check(__wt_snprintf(path, sizeof(path), "%s/%s.wt", g.home, WT_NAME)); + if ((fd = open(path, O_RDWR)) != -1) { testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s.wt %s/SALVAGE.copy/%s.wt.corrupted", g.home, WT_NAME, g.home, WT_NAME)); goto found; @@ -87,9 +88,9 @@ found: offset = mmrand(NULL, 0, (u_int)sb.st_size); len = (size_t)(20 + (sb.st_size / 100) * 2); - testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/SALVAGE.corrupt", g.home)); - if ((fp = fopen(buf, "w")) == NULL) - testutil_die(errno, "salvage-corrupt: open: %s", buf); + testutil_check(__wt_snprintf(path, sizeof(path), "%s/SALVAGE.corrupt", g.home)); + if ((fp = fopen(path, "w")) == NULL) + testutil_die(errno, "salvage-corrupt: open: %s", path); (void)fprintf(fp, "salvage-corrupt: offset %" PRIuMAX ", length %" WT_SIZET_FMT "\n", (uintmax_t)offset, len); fclose_and_clear(&fp); @@ -97,10 +98,10 @@ found: if (lseek(fd, offset, SEEK_SET) == -1) testutil_die(errno, "salvage-corrupt: lseek"); - memset(buf, 'z', sizeof(buf)); + smash = "!!! memory corrupted by format to test salvage "; for (; len > 0; len -= nw) { - nw = (size_t)(len > sizeof(buf) ? sizeof(buf) : len); - if (write(fd, buf, nw) == -1) + nw = (size_t)(len > strlen(smash) ? strlen(smash) : len); + if (write(fd, smash, nw) == -1) testutil_die(errno, "salvage-corrupt: write"); } @@ -150,16 +151,17 @@ wts_salvage(void) free(cmd); /* Salvage, then verify. */ - wts_open(g.home, true, &g.wts_conn); + wts_open(g.home, true, &g.wts_conn, true); salvage(); +#if 0 wts_verify("post-salvage verify"); +#endif wts_close(); /* Corrupt the file randomly, salvage, then verify. */ if (corrupt()) { - wts_open(g.home, true, &g.wts_conn); + wts_open(g.home, true, &g.wts_conn, false); salvage(); - wts_verify("post-corrupt-salvage verify"); wts_close(); } } diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 5e68d4c524b..61d612b1abd 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -279,7 +279,7 @@ main(int argc, char *argv[]) if (g.reopen) wts_reopen(); /* Reopen existing database. */ else { - wts_open(g.home, true, &g.wts_conn); + wts_open(g.home, true, &g.wts_conn, true); wts_init(); TIMED_MAJOR_OP(wts_load()); /* Load and verify initial records */ TIMED_MAJOR_OP(wts_verify("post-bulk verify")); diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 66a319a9982..309b576be63 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -176,7 +176,7 @@ static WT_EVENT_HANDLER event_handler = { * Open a connection to a WiredTiger database. */ void -wts_open(const char *home, bool set_api, WT_CONNECTION **connp) +wts_open(const char *home, bool set_api, WT_CONNECTION **connp, bool allow_verify) { WT_CONNECTION *conn; size_t max; @@ -282,8 +282,10 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) CONFIG_APPEND(p, "]"); #if WIREDTIGER_VERSION_MAJOR >= 10 - if (g.c_verify) + if (g.c_verify && allow_verify) CONFIG_APPEND(p, ",verify_metadata=true"); + else if (!allow_verify) + CONFIG_APPEND(p, ",verify_metadata=false"); #endif /* Extensions. */ diff --git a/src/third_party/wiredtiger/test/suite/test_prepare11.py b/src/third_party/wiredtiger/test/suite/test_prepare11.py new file mode 100644 index 00000000000..4aac11a7ed8 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare11.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +def timestamp_str(t): + return '%x' % t + +# test_prepare11.py +# Test prepare rollback with a reserved update between updates. +class test_prepare11(wttest.WiredTigerTestCase): + conn_config = 'cache_size=2MB,statistics=(all)' + session_config = 'isolation=snapshot' + + def test_prepare_update_rollback(self): + uri = "table:test_prepare11" + self.session.create(uri, 'key_format=S,value_format=S') + self.session.begin_transaction("isolation=snapshot") + + # In the scenario where we have a reserved update in between two updates, the key repeated + # flag won't get set and we'll call resolve prepared op on both prepared updates. + c = self.session.open_cursor(uri, None) + c['key1'] = 'xxxx' + c.set_key('key1') + c.reserve() + c['key1'] = 'yyyy' + self.session.prepare_transaction('prepare_timestamp=10') + self.session.rollback_transaction() |