WT-2541 Add stats showing active readers and writers (#2793)

* WT-2541 Add stats showing active readers and writers

* Fix use of RET_MSG instead of ERR_MSG

* Change where we increment the active r/w stats

* Add "active fsync" stat

* Add new "Thread State" stat section, add fsync_io stat

* KNF

* Make thread stats not backwards breaking. Add atomic stat functions

* Add ThreadState stats to system and evict groups

* KNF

4 files changed, 161 insertions, 98 deletions

diff --git a/src/include/os_fhandle.i b/src/include/os_fhandle.i
index cf790d6bc4d..313bf8eca3f 100644
--- a/src/include/os_fhandle.i
+++ b/src/include/os_fhandle.i
@@ -13,6 +13,7 @@
 static inline int
 __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh, bool block)
 {
+	WT_DECL_RET;
 	WT_FILE_HANDLE *handle;
 
 	WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY));
@@ -21,12 +22,20 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh, bool block)
 	    session, WT_VERB_HANDLEOPS, "%s: handle-sync", fh->handle->name));
 
 	handle = fh->handle;
+	/*
+	 * There is no way to check when the non-blocking sync-file-range is
+	 * complete, but we track the time taken in the call for completeness.
+	 */
+	WT_STAT_FAST_CONN_INCR_ATOMIC(session, fsync_active);
+	WT_STAT_FAST_CONN_INCR(session, fsync_io);
 	if (block)
-		return (handle->fh_sync == NULL ? 0 :
+		ret = (handle->fh_sync == NULL ? 0 :
 		    handle->fh_sync(handle, (WT_SESSION *)session));
 	else
-		return (handle->fh_sync_nowait == NULL ? 0 :
+		ret = (handle->fh_sync_nowait == NULL ? 0 :
 		    handle->fh_sync_nowait(handle, (WT_SESSION *)session));
+	WT_STAT_FAST_CONN_DECR_ATOMIC(session, fsync_active);
+	return (ret);
 }
 
 /*
@@ -92,14 +101,20 @@ static inline int
 __wt_read(
     WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, size_t len, void *buf)
 {
+	WT_DECL_RET;
+
 	WT_RET(__wt_verbose(session, WT_VERB_HANDLEOPS,
 	    "%s: handle-read: %" WT_SIZET_FMT " at %" PRIuMAX,
 	    fh->handle->name, len, (uintmax_t)offset));
 
+	WT_STAT_FAST_CONN_INCR_ATOMIC(session, read_active);
 	WT_STAT_FAST_CONN_INCR(session, read_io);
 
-	return (fh->handle->fh_read(
-	    fh->handle, (WT_SESSION *)session, offset, len, buf));
+	ret = fh->handle->fh_read(
+	    fh->handle, (WT_SESSION *)session, offset, len, buf);
+
+	WT_STAT_FAST_CONN_DECR_ATOMIC(session, read_active);
+	return (ret);
 }
 
 /*
@@ -140,6 +155,8 @@ static inline int
 __wt_write(WT_SESSION_IMPL *session,
     WT_FH *fh, wt_off_t offset, size_t len, const void *buf)
 {
+	WT_DECL_RET;
+
 	WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) ||
 	    WT_STRING_MATCH(fh->name,
 	    WT_SINGLETHREAD, strlen(WT_SINGLETHREAD)));
@@ -148,8 +165,12 @@ __wt_write(WT_SESSION_IMPL *session,
 	    "%s: handle-write: %" WT_SIZET_FMT " at %" PRIuMAX,
 	    fh->handle->name, len, (uintmax_t)offset));
 
+	WT_STAT_FAST_CONN_INCR_ATOMIC(session, write_active);
 	WT_STAT_FAST_CONN_INCR(session, write_io);
 
-	return (fh->handle->fh_write(
-	    fh->handle, (WT_SESSION *)session, offset, len, buf));
+	ret = fh->handle->fh_write(
+	    fh->handle, (WT_SESSION *)session, offset, len, buf);
+
+	WT_STAT_FAST_CONN_DECR_ATOMIC(session, write_active);
+	return (ret);
 }
diff --git a/src/include/stat.h b/src/include/stat.h
index e921d64a0c5..079a9245b07 100644
--- a/src/include/stat.h
+++ b/src/include/stat.h
@@ -144,10 +144,16 @@ __wt_stats_clear(void *stats_arg, int slot)
 
 #define	WT_STAT_DECRV(session, stats, fld, value)			\
 	(stats)[WT_STATS_SLOT_ID(session)]->fld -= (int64_t)(value)
+#define	WT_STAT_DECRV_ATOMIC(session, stats, fld, value)		\
+	__wt_atomic_addi64(						\
+	    &(stats)[WT_STATS_SLOT_ID(session)]->fld, (int64_t)(value))
 #define	WT_STAT_DECR(session, stats, fld)				\
 	WT_STAT_DECRV(session, stats, fld, 1)
 #define	WT_STAT_INCRV(session, stats, fld, value)			\
 	(stats)[WT_STATS_SLOT_ID(session)]->fld += (int64_t)(value)
+#define	WT_STAT_INCRV_ATOMIC(session, stats, fld, value)		\
+	__wt_atomic_subi64(						\
+	    &(stats)[WT_STATS_SLOT_ID(session)]->fld, (int64_t)(value))
 #define	WT_STAT_INCR(session, stats, fld)				\
 	WT_STAT_INCRV(session, stats, fld, 1)
 #define	WT_STAT_SET(session, stats, fld, value) do {			\
@@ -164,12 +170,20 @@ __wt_stats_clear(void *stats_arg, int slot)
 } while (0)
 #define	WT_STAT_FAST_DECR(session, stats, fld)				\
 	WT_STAT_FAST_DECRV(session, stats, fld, 1)
+#define	WT_STAT_FAST_DECRV_ATOMIC(session, stats, fld, value) do {	\
+	if (FLD_ISSET(S2C(session)->stat_flags, WT_CONN_STAT_FAST))	\
+		WT_STAT_DECRV_ATOMIC(session, stats, fld, value);	\
+} while (0)
 #define	WT_STAT_FAST_INCRV(session, stats, fld, value) do {		\
 	if (FLD_ISSET(S2C(session)->stat_flags, WT_CONN_STAT_FAST))	\
 		WT_STAT_INCRV(session, stats, fld, value);		\
 } while (0)
 #define	WT_STAT_FAST_INCR(session, stats, fld)				\
 	WT_STAT_FAST_INCRV(session, stats, fld, 1)
+#define	WT_STAT_FAST_INCRV_ATOMIC(session, stats, fld, value) do {	\
+	if (FLD_ISSET(S2C(session)->stat_flags, WT_CONN_STAT_FAST))	\
+		WT_STAT_INCRV_ATOMIC(session, stats, fld, value);	\
+} while (0)
 #define	WT_STAT_FAST_SET(session, stats, fld, value) do {		\
 	if (FLD_ISSET(S2C(session)->stat_flags, WT_CONN_STAT_FAST))	\
 		WT_STAT_SET(session, stats, fld, value);		\
@@ -180,10 +194,14 @@ __wt_stats_clear(void *stats_arg, int slot)
  */
 #define	WT_STAT_FAST_CONN_DECR(session, fld)				\
 	WT_STAT_FAST_DECR(session, S2C(session)->stats, fld)
+#define	WT_STAT_FAST_CONN_DECR_ATOMIC(session, fld)			\
+	WT_STAT_FAST_DECRV_ATOMIC(session, S2C(session)->stats, fld, 1)
 #define	WT_STAT_FAST_CONN_DECRV(session, fld, value)			\
 	WT_STAT_FAST_DECRV(session, S2C(session)->stats, fld, value)
 #define	WT_STAT_FAST_CONN_INCR(session, fld)				\
 	WT_STAT_FAST_INCR(session, S2C(session)->stats, fld)
+#define	WT_STAT_FAST_CONN_INCR_ATOMIC(session, fld)			\
+	WT_STAT_FAST_INCRV_ATOMIC(session, S2C(session)->stats, fld, 1)
 #define	WT_STAT_FAST_CONN_INCRV(session, fld, value)			\
 	WT_STAT_FAST_INCRV(session, S2C(session)->stats, fld, value)
 #define	WT_STAT_FAST_CONN_SET(session, fld, value)			\
@@ -317,6 +335,7 @@ struct __wt_connection_stats {
 	int64_t cond_wait;
 	int64_t rwlock_read;
 	int64_t rwlock_write;
+	int64_t fsync_io;
 	int64_t read_io;
 	int64_t write_io;
 	int64_t cursor_create;
@@ -383,6 +402,9 @@ struct __wt_connection_stats {
 	int64_t rec_split_stashed_objects;
 	int64_t session_cursor_open;
 	int64_t session_open;
+	int64_t fsync_active;
+	int64_t read_active;
+	int64_t write_active;
 	int64_t page_busy_blocked;
 	int64_t page_forcible_evict_blocked;
 	int64_t page_locked_blocked;
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index 77752d7a7b4..9c189f573ff 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -4385,195 +4385,203 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
 #define	WT_STAT_CONN_RWLOCK_READ			1088
 /*! connection: pthread mutex shared lock write-lock calls */
 #define	WT_STAT_CONN_RWLOCK_WRITE			1089
+/*! connection: total fsync I/Os */
+#define	WT_STAT_CONN_FSYNC_IO				1090
 /*! connection: total read I/Os */
-#define	WT_STAT_CONN_READ_IO				1090
+#define	WT_STAT_CONN_READ_IO				1091
 /*! connection: total write I/Os */
-#define	WT_STAT_CONN_WRITE_IO				1091
+#define	WT_STAT_CONN_WRITE_IO				1092
 /*! cursor: cursor create calls */
-#define	WT_STAT_CONN_CURSOR_CREATE			1092
+#define	WT_STAT_CONN_CURSOR_CREATE			1093
 /*! cursor: cursor insert calls */
-#define	WT_STAT_CONN_CURSOR_INSERT			1093
+#define	WT_STAT_CONN_CURSOR_INSERT			1094
 /*! cursor: cursor next calls */
-#define	WT_STAT_CONN_CURSOR_NEXT			1094
+#define	WT_STAT_CONN_CURSOR_NEXT			1095
 /*! cursor: cursor prev calls */
-#define	WT_STAT_CONN_CURSOR_PREV			1095
+#define	WT_STAT_CONN_CURSOR_PREV			1096
 /*! cursor: cursor remove calls */
-#define	WT_STAT_CONN_CURSOR_REMOVE			1096
+#define	WT_STAT_CONN_CURSOR_REMOVE			1097
 /*! cursor: cursor reset calls */
-#define	WT_STAT_CONN_CURSOR_RESET			1097
+#define	WT_STAT_CONN_CURSOR_RESET			1098
 /*! cursor: cursor restarted searches */
-#define	WT_STAT_CONN_CURSOR_RESTART			1098
+#define	WT_STAT_CONN_CURSOR_RESTART			1099
 /*! cursor: cursor search calls */
-#define	WT_STAT_CONN_CURSOR_SEARCH			1099
+#define	WT_STAT_CONN_CURSOR_SEARCH			1100
 /*! cursor: cursor search near calls */
-#define	WT_STAT_CONN_CURSOR_SEARCH_NEAR			1100
+#define	WT_STAT_CONN_CURSOR_SEARCH_NEAR			1101
 /*! cursor: cursor update calls */
-#define	WT_STAT_CONN_CURSOR_UPDATE			1101
+#define	WT_STAT_CONN_CURSOR_UPDATE			1102
 /*! cursor: truncate calls */
-#define	WT_STAT_CONN_CURSOR_TRUNCATE			1102
+#define	WT_STAT_CONN_CURSOR_TRUNCATE			1103
 /*! data-handle: connection data handles currently active */
-#define	WT_STAT_CONN_DH_CONN_HANDLE_COUNT		1103
+#define	WT_STAT_CONN_DH_CONN_HANDLE_COUNT		1104
 /*! data-handle: connection sweep candidate became referenced */
-#define	WT_STAT_CONN_DH_SWEEP_REF			1104
+#define	WT_STAT_CONN_DH_SWEEP_REF			1105
 /*! data-handle: connection sweep dhandles closed */
-#define	WT_STAT_CONN_DH_SWEEP_CLOSE			1105
+#define	WT_STAT_CONN_DH_SWEEP_CLOSE			1106
 /*! data-handle: connection sweep dhandles removed from hash list */
-#define	WT_STAT_CONN_DH_SWEEP_REMOVE			1106
+#define	WT_STAT_CONN_DH_SWEEP_REMOVE			1107
 /*! data-handle: connection sweep time-of-death sets */
-#define	WT_STAT_CONN_DH_SWEEP_TOD			1107
+#define	WT_STAT_CONN_DH_SWEEP_TOD			1108
 /*! data-handle: connection sweeps */
-#define	WT_STAT_CONN_DH_SWEEPS				1108
+#define	WT_STAT_CONN_DH_SWEEPS				1109
 /*! data-handle: session dhandles swept */
-#define	WT_STAT_CONN_DH_SESSION_HANDLES			1109
+#define	WT_STAT_CONN_DH_SESSION_HANDLES			1110
 /*! data-handle: session sweep attempts */
-#define	WT_STAT_CONN_DH_SESSION_SWEEPS			1110
+#define	WT_STAT_CONN_DH_SESSION_SWEEPS			1111
 /*! log: busy returns attempting to switch slots */
-#define	WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY		1111
+#define	WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY		1112
 /*! log: consolidated slot closures */
-#define	WT_STAT_CONN_LOG_SLOT_CLOSES			1112
+#define	WT_STAT_CONN_LOG_SLOT_CLOSES			1113
 /*! log: consolidated slot join races */
-#define	WT_STAT_CONN_LOG_SLOT_RACES			1113
+#define	WT_STAT_CONN_LOG_SLOT_RACES			1114
 /*! log: consolidated slot join transitions */
-#define	WT_STAT_CONN_LOG_SLOT_TRANSITIONS		1114
+#define	WT_STAT_CONN_LOG_SLOT_TRANSITIONS		1115
 /*! log: consolidated slot joins */
-#define	WT_STAT_CONN_LOG_SLOT_JOINS			1115
+#define	WT_STAT_CONN_LOG_SLOT_JOINS			1116
 /*! log: consolidated slot unbuffered writes */
-#define	WT_STAT_CONN_LOG_SLOT_UNBUFFERED		1116
+#define	WT_STAT_CONN_LOG_SLOT_UNBUFFERED		1117
 /*! log: log bytes of payload data */
-#define	WT_STAT_CONN_LOG_BYTES_PAYLOAD			1117
+#define	WT_STAT_CONN_LOG_BYTES_PAYLOAD			1118
 /*! log: log bytes written */
-#define	WT_STAT_CONN_LOG_BYTES_WRITTEN			1118
+#define	WT_STAT_CONN_LOG_BYTES_WRITTEN			1119
 /*! log: log files manually zero-filled */
-#define	WT_STAT_CONN_LOG_ZERO_FILLS			1119
+#define	WT_STAT_CONN_LOG_ZERO_FILLS			1120
 /*! log: log flush operations */
-#define	WT_STAT_CONN_LOG_FLUSH				1120
+#define	WT_STAT_CONN_LOG_FLUSH				1121
 /*! log: log force write operations */
-#define	WT_STAT_CONN_LOG_FORCE_WRITE			1121
+#define	WT_STAT_CONN_LOG_FORCE_WRITE			1122
 /*! log: log force write operations skipped */
-#define	WT_STAT_CONN_LOG_FORCE_WRITE_SKIP		1122
+#define	WT_STAT_CONN_LOG_FORCE_WRITE_SKIP		1123
 /*! log: log records compressed */
-#define	WT_STAT_CONN_LOG_COMPRESS_WRITES		1123
+#define	WT_STAT_CONN_LOG_COMPRESS_WRITES		1124
 /*! log: log records not compressed */
-#define	WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS		1124
+#define	WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS		1125
 /*! log: log records too small to compress */
-#define	WT_STAT_CONN_LOG_COMPRESS_SMALL			1125
+#define	WT_STAT_CONN_LOG_COMPRESS_SMALL			1126
 /*! log: log release advances write LSN */
-#define	WT_STAT_CONN_LOG_RELEASE_WRITE_LSN		1126
+#define	WT_STAT_CONN_LOG_RELEASE_WRITE_LSN		1127
 /*! log: log scan operations */
-#define	WT_STAT_CONN_LOG_SCANS				1127
+#define	WT_STAT_CONN_LOG_SCANS				1128
 /*! log: log scan records requiring two reads */
-#define	WT_STAT_CONN_LOG_SCAN_REREADS			1128
+#define	WT_STAT_CONN_LOG_SCAN_REREADS			1129
 /*! log: log server thread advances write LSN */
-#define	WT_STAT_CONN_LOG_WRITE_LSN			1129
+#define	WT_STAT_CONN_LOG_WRITE_LSN			1130
 /*! log: log server thread write LSN walk skipped */
-#define	WT_STAT_CONN_LOG_WRITE_LSN_SKIP			1130
+#define	WT_STAT_CONN_LOG_WRITE_LSN_SKIP			1131
 /*! log: log sync operations */
-#define	WT_STAT_CONN_LOG_SYNC				1131
+#define	WT_STAT_CONN_LOG_SYNC				1132
 /*! log: log sync time duration (usecs) */
-#define	WT_STAT_CONN_LOG_SYNC_DURATION			1132
+#define	WT_STAT_CONN_LOG_SYNC_DURATION			1133
 /*! log: log sync_dir operations */
-#define	WT_STAT_CONN_LOG_SYNC_DIR			1133
+#define	WT_STAT_CONN_LOG_SYNC_DIR			1134
 /*! log: log sync_dir time duration (usecs) */
-#define	WT_STAT_CONN_LOG_SYNC_DIR_DURATION		1134
+#define	WT_STAT_CONN_LOG_SYNC_DIR_DURATION		1135
 /*! log: log write operations */
-#define	WT_STAT_CONN_LOG_WRITES				1135
+#define	WT_STAT_CONN_LOG_WRITES				1136
 /*! log: logging bytes consolidated */
-#define	WT_STAT_CONN_LOG_SLOT_CONSOLIDATED		1136
+#define	WT_STAT_CONN_LOG_SLOT_CONSOLIDATED		1137
 /*! log: maximum log file size */
-#define	WT_STAT_CONN_LOG_MAX_FILESIZE			1137
+#define	WT_STAT_CONN_LOG_MAX_FILESIZE			1138
 /*! log: number of pre-allocated log files to create */
-#define	WT_STAT_CONN_LOG_PREALLOC_MAX			1138
+#define	WT_STAT_CONN_LOG_PREALLOC_MAX			1139
 /*! log: pre-allocated log files not ready and missed */
-#define	WT_STAT_CONN_LOG_PREALLOC_MISSED		1139
+#define	WT_STAT_CONN_LOG_PREALLOC_MISSED		1140
 /*! log: pre-allocated log files prepared */
-#define	WT_STAT_CONN_LOG_PREALLOC_FILES			1140
+#define	WT_STAT_CONN_LOG_PREALLOC_FILES			1141
 /*! log: pre-allocated log files used */
-#define	WT_STAT_CONN_LOG_PREALLOC_USED			1141
+#define	WT_STAT_CONN_LOG_PREALLOC_USED			1142
 /*! log: records processed by log scan */
-#define	WT_STAT_CONN_LOG_SCAN_RECORDS			1142
+#define	WT_STAT_CONN_LOG_SCAN_RECORDS			1143
 /*! log: total in-memory size of compressed records */
-#define	WT_STAT_CONN_LOG_COMPRESS_MEM			1143
+#define	WT_STAT_CONN_LOG_COMPRESS_MEM			1144
 /*! log: total log buffer size */
-#define	WT_STAT_CONN_LOG_BUFFER_SIZE			1144
+#define	WT_STAT_CONN_LOG_BUFFER_SIZE			1145
 /*! log: total size of compressed records */
-#define	WT_STAT_CONN_LOG_COMPRESS_LEN			1145
+#define	WT_STAT_CONN_LOG_COMPRESS_LEN			1146
 /*! log: written slots coalesced */
-#define	WT_STAT_CONN_LOG_SLOT_COALESCED			1146
+#define	WT_STAT_CONN_LOG_SLOT_COALESCED			1147
 /*! log: yields waiting for previous log file close */
-#define	WT_STAT_CONN_LOG_CLOSE_YIELDS			1147
+#define	WT_STAT_CONN_LOG_CLOSE_YIELDS			1148
 /*! reconciliation: fast-path pages deleted */
-#define	WT_STAT_CONN_REC_PAGE_DELETE_FAST		1148
+#define	WT_STAT_CONN_REC_PAGE_DELETE_FAST		1149
 /*! reconciliation: page reconciliation calls */
-#define	WT_STAT_CONN_REC_PAGES				1149
+#define	WT_STAT_CONN_REC_PAGES				1150
 /*! reconciliation: page reconciliation calls for eviction */
-#define	WT_STAT_CONN_REC_PAGES_EVICTION			1150
+#define	WT_STAT_CONN_REC_PAGES_EVICTION			1151
 /*! reconciliation: pages deleted */
-#define	WT_STAT_CONN_REC_PAGE_DELETE			1151
+#define	WT_STAT_CONN_REC_PAGE_DELETE			1152
 /*! reconciliation: split bytes currently awaiting free */
-#define	WT_STAT_CONN_REC_SPLIT_STASHED_BYTES		1152
+#define	WT_STAT_CONN_REC_SPLIT_STASHED_BYTES		1153
 /*! reconciliation: split objects currently awaiting free */
-#define	WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS		1153
+#define	WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS		1154
 /*! session: open cursor count */
-#define	WT_STAT_CONN_SESSION_CURSOR_OPEN		1154
+#define	WT_STAT_CONN_SESSION_CURSOR_OPEN		1155
 /*! session: open session count */
-#define	WT_STAT_CONN_SESSION_OPEN			1155
+#define	WT_STAT_CONN_SESSION_OPEN			1156
+/*! thread-state: active filesystem fsync calls */
+#define	WT_STAT_CONN_FSYNC_ACTIVE			1157
+/*! thread-state: active filesystem read calls */
+#define	WT_STAT_CONN_READ_ACTIVE			1158
+/*! thread-state: active filesystem write calls */
+#define	WT_STAT_CONN_WRITE_ACTIVE			1159
 /*! thread-yield: page acquire busy blocked */
-#define	WT_STAT_CONN_PAGE_BUSY_BLOCKED			1156
+#define	WT_STAT_CONN_PAGE_BUSY_BLOCKED			1160
 /*! thread-yield: page acquire eviction blocked */
-#define	WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED	1157
+#define	WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED	1161
 /*! thread-yield: page acquire locked blocked */
-#define	WT_STAT_CONN_PAGE_LOCKED_BLOCKED		1158
+#define	WT_STAT_CONN_PAGE_LOCKED_BLOCKED		1162
 /*! thread-yield: page acquire read blocked */
-#define	WT_STAT_CONN_PAGE_READ_BLOCKED			1159
+#define	WT_STAT_CONN_PAGE_READ_BLOCKED			1163
 /*! thread-yield: page acquire time sleeping (usecs) */
-#define	WT_STAT_CONN_PAGE_SLEEP				1160
+#define	WT_STAT_CONN_PAGE_SLEEP				1164
 /*! transaction: number of named snapshots created */
-#define	WT_STAT_CONN_TXN_SNAPSHOTS_CREATED		1161
+#define	WT_STAT_CONN_TXN_SNAPSHOTS_CREATED		1165
 /*! transaction: number of named snapshots dropped */
-#define	WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED		1162
+#define	WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED		1166
 /*! transaction: transaction begins */
-#define	WT_STAT_CONN_TXN_BEGIN				1163
+#define	WT_STAT_CONN_TXN_BEGIN				1167
 /*! transaction: transaction checkpoint currently running */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_RUNNING		1164
+#define	WT_STAT_CONN_TXN_CHECKPOINT_RUNNING		1168
 /*! transaction: transaction checkpoint generation */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_GENERATION		1165
+#define	WT_STAT_CONN_TXN_CHECKPOINT_GENERATION		1169
 /*! transaction: transaction checkpoint max time (msecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX		1166
+#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX		1170
 /*! transaction: transaction checkpoint min time (msecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN		1167
+#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN		1171
 /*! transaction: transaction checkpoint most recent time (msecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT		1168
+#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT		1172
 /*! transaction: transaction checkpoint total time (msecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL		1169
+#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL		1173
 /*! transaction: transaction checkpoints */
-#define	WT_STAT_CONN_TXN_CHECKPOINT			1170
+#define	WT_STAT_CONN_TXN_CHECKPOINT			1174
 /*! transaction: transaction failures due to cache overflow */
-#define	WT_STAT_CONN_TXN_FAIL_CACHE			1171
+#define	WT_STAT_CONN_TXN_FAIL_CACHE			1175
 /*! transaction: transaction fsync calls for checkpoint after allocating
  * the transaction ID */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST		1172
+#define	WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST		1176
 /*! transaction: transaction fsync calls for checkpoint before allocating
  * the transaction ID */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_PRE		1173
+#define	WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_PRE		1177
 /*! transaction: transaction fsync duration for checkpoint after
  * allocating the transaction ID (usecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION	1174
+#define	WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION	1178
 /*! transaction: transaction fsync duration for checkpoint before
  * allocating the transaction ID (usecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_PRE_DURATION	1175
+#define	WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_PRE_DURATION	1179
 /*! transaction: transaction range of IDs currently pinned */
-#define	WT_STAT_CONN_TXN_PINNED_RANGE			1176
+#define	WT_STAT_CONN_TXN_PINNED_RANGE			1180
 /*! transaction: transaction range of IDs currently pinned by a checkpoint */
-#define	WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE	1177
+#define	WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE	1181
 /*! transaction: transaction range of IDs currently pinned by named
  * snapshots */
-#define	WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE		1178
+#define	WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE		1182
 /*! transaction: transaction sync calls */
-#define	WT_STAT_CONN_TXN_SYNC				1179
+#define	WT_STAT_CONN_TXN_SYNC				1183
 /*! transaction: transactions committed */
-#define	WT_STAT_CONN_TXN_COMMIT				1180
+#define	WT_STAT_CONN_TXN_COMMIT				1184
 /*! transaction: transactions rolled back */
-#define	WT_STAT_CONN_TXN_ROLLBACK			1181
+#define	WT_STAT_CONN_TXN_ROLLBACK			1185
 
 /*!
  * @}
diff --git a/src/support/stat.c b/src/support/stat.c
index 0f555a77ccb..661f155e932 100644
--- a/src/support/stat.c
+++ b/src/support/stat.c
@@ -604,6 +604,7 @@ static const char * const __stats_connection_desc[] = {
 	"connection: pthread mutex condition wait calls",
 	"connection: pthread mutex shared lock read-lock calls",
 	"connection: pthread mutex shared lock write-lock calls",
+	"connection: total fsync I/Os",
 	"connection: total read I/Os",
 	"connection: total write I/Os",
 	"cursor: cursor create calls",
@@ -670,6 +671,9 @@ static const char * const __stats_connection_desc[] = {
 	"reconciliation: split objects currently awaiting free",
 	"session: open cursor count",
 	"session: open session count",
+	"thread-state: active filesystem fsync calls",
+	"thread-state: active filesystem read calls",
+	"thread-state: active filesystem write calls",
 	"thread-yield: page acquire busy blocked",
 	"thread-yield: page acquire eviction blocked",
 	"thread-yield: page acquire locked blocked",
@@ -816,6 +820,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats)
 	stats->cond_wait = 0;
 	stats->rwlock_read = 0;
 	stats->rwlock_write = 0;
+	stats->fsync_io = 0;
 	stats->read_io = 0;
 	stats->write_io = 0;
 	stats->cursor_create = 0;
@@ -882,6 +887,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats)
 		/* not clearing rec_split_stashed_objects */
 		/* not clearing session_cursor_open */
 		/* not clearing session_open */
+		/* not clearing fsync_active */
+		/* not clearing read_active */
+		/* not clearing write_active */
 	stats->page_busy_blocked = 0;
 	stats->page_forcible_evict_blocked = 0;
 	stats->page_locked_blocked = 0;
@@ -1046,6 +1054,7 @@ __wt_stat_connection_aggregate(
 	to->cond_wait += WT_STAT_READ(from, cond_wait);
 	to->rwlock_read += WT_STAT_READ(from, rwlock_read);
 	to->rwlock_write += WT_STAT_READ(from, rwlock_write);
+	to->fsync_io += WT_STAT_READ(from, fsync_io);
 	to->read_io += WT_STAT_READ(from, read_io);
 	to->write_io += WT_STAT_READ(from, write_io);
 	to->cursor_create += WT_STAT_READ(from, cursor_create);
@@ -1118,6 +1127,9 @@ __wt_stat_connection_aggregate(
 	    WT_STAT_READ(from, rec_split_stashed_objects);
 	to->session_cursor_open += WT_STAT_READ(from, session_cursor_open);
 	to->session_open += WT_STAT_READ(from, session_open);
+	to->fsync_active += WT_STAT_READ(from, fsync_active);
+	to->read_active += WT_STAT_READ(from, read_active);
+	to->write_active += WT_STAT_READ(from, write_active);
 	to->page_busy_blocked += WT_STAT_READ(from, page_busy_blocked);
 	to->page_forcible_evict_blocked +=
 	    WT_STAT_READ(from, page_forcible_evict_blocked);