From 534703070f379fff56efd1a600d18395816ed703 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Fri, 15 Feb 2019 13:22:17 +1100 Subject: Import wiredtiger: 94c514c558b60a2e00d4926c7535d062fce5a901 from branch mongodb-4.2 ref: 0c6ba8d8be..94c514c558 for: 4.1.9 WT-4577 Fix LSM deadlock detected by test/format WT-4583 Collect read/write stats in throttling even if not using capacity --- src/third_party/wiredtiger/dist/stat_data.py | 10 +++++----- src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/src/conn/conn_capacity.c | 11 +++++------ src/third_party/wiredtiger/src/include/stat.h | 2 +- .../wiredtiger/src/include/wiredtiger.in | 22 +++++++++++----------- src/third_party/wiredtiger/src/lsm/lsm_work_unit.c | 16 +--------------- src/third_party/wiredtiger/src/support/stat.c | 14 +++++++------- 7 files changed, 31 insertions(+), 46 deletions(-) (limited to 'src/third_party/wiredtiger') diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 9bca52f402a..4fda3e681b7 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -301,11 +301,11 @@ connection_stats = [ ########################################## # Capacity statistics ########################################## - CapacityStat('capacity_bytes_ckpt', 'throttled bytes written for checkpoint'), - CapacityStat('capacity_bytes_evict', 'throttled bytes written for eviction'), - CapacityStat('capacity_bytes_log', 'throttled bytes written for log'), - CapacityStat('capacity_bytes_read', 'throttled bytes read'), - CapacityStat('capacity_bytes_written', 'throttled bytes written total'), + CapacityStat('capacity_bytes_ckpt', 'bytes written for checkpoint'), + CapacityStat('capacity_bytes_evict', 'bytes written for eviction'), + CapacityStat('capacity_bytes_log', 'bytes written for log'), + CapacityStat('capacity_bytes_read', 'bytes read'), + CapacityStat('capacity_bytes_written', 'bytes written total'), CapacityStat('capacity_threshold', 'threshold to call fsync'), CapacityStat('capacity_time_ckpt', 'time waiting during checkpoint (usecs)'), CapacityStat('capacity_time_evict', 'time waiting during eviction (usecs)'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index b522dcbe4b9..c750b8d63ce 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "0c6ba8d8be02dd34a46c3e9533971f1739b6ad8e", + "commit": "94c514c558b60a2e00d4926c7535d062fce5a901", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.2" diff --git a/src/third_party/wiredtiger/src/conn/conn_capacity.c b/src/third_party/wiredtiger/src/conn/conn_capacity.c index 0dd6a8c3c6d..3a6f02a5d02 100644 --- a/src/third_party/wiredtiger/src/conn/conn_capacity.c +++ b/src/third_party/wiredtiger/src/conn/conn_capacity.c @@ -306,10 +306,6 @@ __wt_capacity_throttle(WT_SESSION_IMPL *session, uint64_t bytes, conn = S2C(session); cap = &conn->capacity; - /* If not using capacity there's nothing to do. */ - if (cap->total == 0) - return; - capacity = steal_capacity = 0; reservation = steal = NULL; switch (type) { @@ -317,16 +313,19 @@ __wt_capacity_throttle(WT_SESSION_IMPL *session, uint64_t bytes, capacity = cap->ckpt; reservation = &cap->reservation_ckpt; WT_STAT_CONN_INCRV(session, capacity_bytes_ckpt, bytes); + WT_STAT_CONN_INCRV(session, capacity_bytes_written, bytes); break; case WT_THROTTLE_EVICT: capacity = cap->evict; reservation = &cap->reservation_evict; WT_STAT_CONN_INCRV(session, capacity_bytes_evict, bytes); + WT_STAT_CONN_INCRV(session, capacity_bytes_written, bytes); break; case WT_THROTTLE_LOG: capacity = cap->log; reservation = &cap->reservation_log; WT_STAT_CONN_INCRV(session, capacity_bytes_log, bytes); + WT_STAT_CONN_INCRV(session, capacity_bytes_written, bytes); break; case WT_THROTTLE_READ: capacity = cap->read; @@ -342,7 +341,8 @@ __wt_capacity_throttle(WT_SESSION_IMPL *session, uint64_t bytes, * at some point in the future. If this subsystem is not throttled * there's nothing to do. */ - if (capacity == 0 || F_ISSET(conn, WT_CONN_RECOVERING)) + if (cap->total == 0 || capacity == 0 || + F_ISSET(conn, WT_CONN_RECOVERING)) return; /* @@ -352,7 +352,6 @@ __wt_capacity_throttle(WT_SESSION_IMPL *session, uint64_t bytes, */ if (type != WT_THROTTLE_READ) { (void)__wt_atomic_addv64(&cap->written, bytes); - WT_STAT_CONN_INCRV(session, capacity_bytes_written, bytes); __capacity_signal(session); } diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 40dc8cf695e..f5d1075581b 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -456,12 +456,12 @@ struct __wt_connection_stats { int64_t fsync_all_fh_total; int64_t fsync_all_fh; int64_t fsync_all_time; - int64_t capacity_threshold; int64_t capacity_bytes_read; int64_t capacity_bytes_ckpt; int64_t capacity_bytes_evict; int64_t capacity_bytes_log; int64_t capacity_bytes_written; + int64_t capacity_threshold; int64_t capacity_time_total; int64_t capacity_time_ckpt; int64_t capacity_time_evict; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 1ac4de23044..332bd58f7e6 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -5244,18 +5244,18 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_FSYNC_ALL_FH 1130 /*! capacity: background fsync time (msecs) */ #define WT_STAT_CONN_FSYNC_ALL_TIME 1131 +/*! capacity: bytes read */ +#define WT_STAT_CONN_CAPACITY_BYTES_READ 1132 +/*! capacity: bytes written for checkpoint */ +#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1133 +/*! capacity: bytes written for eviction */ +#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1134 +/*! capacity: bytes written for log */ +#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1135 +/*! capacity: bytes written total */ +#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1136 /*! capacity: threshold to call fsync */ -#define WT_STAT_CONN_CAPACITY_THRESHOLD 1132 -/*! capacity: throttled bytes read */ -#define WT_STAT_CONN_CAPACITY_BYTES_READ 1133 -/*! capacity: throttled bytes written for checkpoint */ -#define WT_STAT_CONN_CAPACITY_BYTES_CKPT 1134 -/*! capacity: throttled bytes written for eviction */ -#define WT_STAT_CONN_CAPACITY_BYTES_EVICT 1135 -/*! capacity: throttled bytes written for log */ -#define WT_STAT_CONN_CAPACITY_BYTES_LOG 1136 -/*! capacity: throttled bytes written total */ -#define WT_STAT_CONN_CAPACITY_BYTES_WRITTEN 1137 +#define WT_STAT_CONN_CAPACITY_THRESHOLD 1137 /*! capacity: time waiting due to total capacity (usecs) */ #define WT_STAT_CONN_CAPACITY_TIME_TOTAL 1138 /*! capacity: time waiting during checkpoint (usecs) */ diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index 5b91aa09db2..aa23e2f3c1f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -818,11 +818,8 @@ err: /* Flush the metadata unless the system is in panic */ int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { - WT_CONNECTION_IMPL *conn; WT_DECL_RET; - conn = S2C(session); - if (lsm_tree->nold_chunks == 0) return (0); @@ -833,18 +830,7 @@ __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (!__wt_atomic_cas32(&lsm_tree->freeing_old_chunks, 0, 1)) return (0); - /* - * Don't remove files if a hot backup is in progress. - * - * The schema lock protects the set of live files, this check prevents - * us from removing a file that hot backup already knows about. - */ - if (!conn->hot_backup) { - __wt_readlock(session, &conn->hot_backup_lock); - if (!conn->hot_backup) - ret = __lsm_free_chunks(session, lsm_tree); - __wt_readunlock(session, &conn->hot_backup_lock); - } + ret = __lsm_free_chunks(session, lsm_tree); lsm_tree->freeing_old_chunks = 0; return (ret); diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 3d5ca2d6a16..4a7b50d72e1 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -885,12 +885,12 @@ static const char * const __stats_connection_desc[] = { "capacity: background fsync file handles considered", "capacity: background fsync file handles synced", "capacity: background fsync time (msecs)", + "capacity: bytes read", + "capacity: bytes written for checkpoint", + "capacity: bytes written for eviction", + "capacity: bytes written for log", + "capacity: bytes written total", "capacity: threshold to call fsync", - "capacity: throttled bytes read", - "capacity: throttled bytes written for checkpoint", - "capacity: throttled bytes written for eviction", - "capacity: throttled bytes written for log", - "capacity: throttled bytes written total", "capacity: time waiting due to total capacity (usecs)", "capacity: time waiting during checkpoint (usecs)", "capacity: time waiting during eviction (usecs)", @@ -1316,12 +1316,12 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->fsync_all_fh_total = 0; stats->fsync_all_fh = 0; /* not clearing fsync_all_time */ - stats->capacity_threshold = 0; stats->capacity_bytes_read = 0; stats->capacity_bytes_ckpt = 0; stats->capacity_bytes_evict = 0; stats->capacity_bytes_log = 0; stats->capacity_bytes_written = 0; + stats->capacity_threshold = 0; stats->capacity_time_total = 0; stats->capacity_time_ckpt = 0; stats->capacity_time_evict = 0; @@ -1793,13 +1793,13 @@ __wt_stat_connection_aggregate( to->fsync_all_fh_total += WT_STAT_READ(from, fsync_all_fh_total); to->fsync_all_fh += WT_STAT_READ(from, fsync_all_fh); to->fsync_all_time += WT_STAT_READ(from, fsync_all_time); - to->capacity_threshold += WT_STAT_READ(from, capacity_threshold); to->capacity_bytes_read += WT_STAT_READ(from, capacity_bytes_read); to->capacity_bytes_ckpt += WT_STAT_READ(from, capacity_bytes_ckpt); to->capacity_bytes_evict += WT_STAT_READ(from, capacity_bytes_evict); to->capacity_bytes_log += WT_STAT_READ(from, capacity_bytes_log); to->capacity_bytes_written += WT_STAT_READ(from, capacity_bytes_written); + to->capacity_threshold += WT_STAT_READ(from, capacity_threshold); to->capacity_time_total += WT_STAT_READ(from, capacity_time_total); to->capacity_time_ckpt += WT_STAT_READ(from, capacity_time_ckpt); to->capacity_time_evict += WT_STAT_READ(from, capacity_time_evict); -- cgit v1.2.1