WT-2610 Reduce hazard pointer array size. (#2731)

* WT-2610 Reduce hazard pointer array size.

* Add session_count_idle and workload to wtperf.

* Allocate session array outside the loop.

* KNF

* Remove diagnostic call to check hazard pointers.

* Add a few hazard related statistics.

* Revert change.  Reinstate diagnostic hazard page check.

12 files changed, 219 insertions, 136 deletions

diff --git a/bench/wtperf/runners/evict-btree-stress.wtperf b/bench/wtperf/runners/evict-btree-stress.wtperf
new file mode 100644
index 00000000000..740fb88c050
--- /dev/null
+++ b/bench/wtperf/runners/evict-btree-stress.wtperf
@@ -0,0 +1,12 @@
+# wtperf options file: evict btree configuration
+conn_config="cache_size=50M,eviction=(threads_max=4)"
+table_config="type=file"
+icount=10000000
+report_interval=5
+run_time=120
+populate_threads=1
+threads=((count=16,reads=1))
+# Add throughput/latency monitoring
+max_latency=2000
+sample_interval=5
+session_count_idle=100
diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c
index 9d57bdcf6b0..1a3d98d3e3e 100644
--- a/bench/wtperf/wtperf.c
+++ b/bench/wtperf/wtperf.c
@@ -1631,6 +1631,8 @@ execute_workload(CONFIG *cfg)
 {
 	CONFIG_THREAD *threads;
 	WORKLOAD *workp;
+	WT_CONNECTION *conn;
+	WT_SESSION **sessions;
 	pthread_t idle_table_cycle_thread;
 	uint64_t last_ckpts, last_inserts, last_reads, last_truncates;
 	uint64_t last_updates;
@@ -1647,6 +1649,8 @@ execute_workload(CONFIG *cfg)
 	last_updates = 0;
 	ret = 0;
 
+	sessions = NULL;
+
 	/* Start cycling idle tables. */
 	if ((ret = start_idle_table_cycle(cfg, &idle_table_cycle_thread)) != 0)
 		return (ret);
@@ -1664,6 +1668,18 @@ execute_workload(CONFIG *cfg)
 	} else
 		pfunc = worker;
 
+	if (cfg->session_count_idle != 0) {
+		sessions = dcalloc((size_t)cfg->session_count_idle,
+		    sizeof(WT_SESSION *));
+		conn = cfg->conn;
+		for (i = 0; i < cfg->session_count_idle; ++i)
+			if ((ret = conn->open_session(
+			    conn, NULL, cfg->sess_config, &sessions[i])) != 0) {
+				lprintf(cfg, ret, 0,
+				    "execute_workload: idle open_session");
+				goto err;
+			}
+	}
 	/* Start each workload. */
 	for (threads = cfg->workers, i = 0,
 	    workp = cfg->workload; i < cfg->workload_cnt; ++i, ++workp) {
@@ -1758,6 +1774,7 @@ err:	cfg->stop = 1;
 	if (ret == 0 && cfg->drop_tables && (ret = drop_all_tables(cfg)) != 0)
 		lprintf(cfg, ret, 0, "Drop tables failed.");
 
+	free(sessions);
 	/* Report if any worker threads didn't finish. */
 	if (cfg->error != 0) {
 		lprintf(cfg, WT_ERROR, 0,
@@ -2170,15 +2187,15 @@ int
 main(int argc, char *argv[])
 {
 	CONFIG *cfg, _cfg;
-	size_t req_len;
+	size_t req_len, sreq_len;
 	int ch, monitor_set, ret;
 	const char *opts = "C:H:h:m:O:o:T:";
 	const char *config_opts;
-	char *cc_buf, *tc_buf, *user_cconfig, *user_tconfig;
+	char *cc_buf, *sess_cfg, *tc_buf, *user_cconfig, *user_tconfig;
 
 	monitor_set = ret = 0;
 	config_opts = NULL;
-	cc_buf = tc_buf = user_cconfig = user_tconfig = NULL;
+	cc_buf = sess_cfg = tc_buf = user_cconfig = user_tconfig = NULL;
 
 	/* Setup the default configuration values. */
 	cfg = &_cfg;
@@ -2317,7 +2334,8 @@ main(int argc, char *argv[])
 
 	/* Concatenate non-default configuration strings. */
 	if (cfg->verbose > 1 || user_cconfig != NULL ||
-	    cfg->compress_ext != NULL || cfg->async_config != NULL) {
+	    cfg->session_count_idle > 0 || cfg->compress_ext != NULL ||
+	    cfg->async_config != NULL) {
 		req_len = strlen(cfg->conn_config) + strlen(debug_cconfig) + 3;
 		if (user_cconfig != NULL)
 			req_len += strlen(user_cconfig);
@@ -2325,16 +2343,26 @@ main(int argc, char *argv[])
 			req_len += strlen(cfg->async_config);
 		if (cfg->compress_ext != NULL)
 			req_len += strlen(cfg->compress_ext);
+		if (cfg->session_count_idle > 0) {
+			sreq_len = strlen(",session_max=") + 6;
+			req_len += sreq_len;
+			sess_cfg = dcalloc(sreq_len, 1);
+			snprintf(sess_cfg, sreq_len,
+			    ",session_max=%" PRIu32,
+			    cfg->session_count_idle + cfg->workers_cnt +
+			    cfg->populate_threads + 10);
+		}
 		cc_buf = dcalloc(req_len, 1);
 		/*
 		 * This is getting hard to parse.
 		 */
-		snprintf(cc_buf, req_len, "%s%s%s%s%s%s%s",
+		snprintf(cc_buf, req_len, "%s%s%s%s%s%s%s%s",
 		    cfg->conn_config,
 		    cfg->async_config ? cfg->async_config : "",
 		    cfg->compress_ext ? cfg->compress_ext : "",
 		    cfg->verbose > 1 ? ",": "",
 		    cfg->verbose > 1 ? debug_cconfig : "",
+		    sess_cfg ? sess_cfg : "",
 		    user_cconfig ? ",": "",
 		    user_cconfig ? user_cconfig : "");
 		if ((ret = config_opt_str(cfg, "conn_config", cc_buf)) != 0)
@@ -2410,6 +2438,7 @@ einval:		ret = EINVAL;
 
 err:	config_free(cfg);
 	free(cc_buf);
+	free(sess_cfg);
 	free(tc_buf);
 	free(user_cconfig);
 	free(user_tconfig);
diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i
index b5e274a17c2..2afd20f777f 100644
--- a/bench/wtperf/wtperf_opt.i
+++ b/bench/wtperf/wtperf_opt.i
@@ -163,6 +163,8 @@ DEF_OPT_AS_UINT32(sample_rate, 50,
     "how often the latency of operations is measured. One for every operation,"
     "two for every second operation, three for every third operation etc.")
 DEF_OPT_AS_CONFIG_STRING(sess_config, "", "session configuration string")
+DEF_OPT_AS_UINT32(session_count_idle, 0,
+    "number of idle sessions to create. Default 0.")
 DEF_OPT_AS_CONFIG_STRING(table_config,
     "key_format=S,value_format=S,type=lsm,exclusive=true,"
     "allocation_size=4kb,internal_page_max=64kb,leaf_page_max=4kb,"
diff --git a/dist/stat_data.py b/dist/stat_data.py
index 0486d94e278..48066c11700 100644
--- a/dist/stat_data.py
+++ b/dist/stat_data.py
@@ -187,6 +187,9 @@ connection_stats = [
     CacheStat('cache_eviction_split_leaf', 'leaf pages split during eviction'),
     CacheStat('cache_eviction_walk', 'pages walked for eviction'),
     CacheStat('cache_eviction_worker_evicting', 'eviction worker thread evicting pages'),
+    CacheStat('cache_hazard_checks', 'hazard pointer check calls'),
+    CacheStat('cache_hazard_max', 'hazard pointer maximum array length', 'max_aggregate,no_scale'),
+    CacheStat('cache_hazard_walks', 'hazard pointer check entries walked'),
     CacheStat('cache_inmem_split', 'in-memory page splits'),
     CacheStat('cache_inmem_splittable', 'in-memory page passed criteria to be split'),
     CacheStat('cache_lookaside_insert', 'lookaside table insert calls'),
diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox
index 6d8dcab8f65..e06272d117c 100644
--- a/src/docs/wtperf.dox
+++ b/src/docs/wtperf.dox
@@ -232,6 +232,8 @@ operation,two for every second operation, three for every third
 operation etc.
 @par sess_config (string, default=)
 session configuration string
+@par session_count_idle (unsigned int, default=0)
+number of idle sessions to create. Default 0.
 @par table_config (string, default=key_format=S,value_format=S,type=lsm,exclusive=true,allocation_size=4kb,internal_page_max=64kb,leaf_page_max=4kb,split_pct=100)
 table configuration string
 @par table_count (unsigned int, default=1)
diff --git a/src/include/btree.i b/src/include/btree.i
index 4c8166ca6a6..e0102a11511 100644
--- a/src/include/btree.i
+++ b/src/include/btree.i
@@ -1360,7 +1360,7 @@ __wt_page_hazard_check(WT_SESSION_IMPL *session, WT_PAGE *page)
 	WT_CONNECTION_IMPL *conn;
 	WT_HAZARD *hp;
 	WT_SESSION_IMPL *s;
-	uint32_t i, hazard_size, session_cnt;
+	uint32_t i, j, hazard_size, max, session_cnt;
 
 	conn = S2C(session);
 
@@ -1372,15 +1372,28 @@ __wt_page_hazard_check(WT_SESSION_IMPL *session, WT_PAGE *page)
 	 * come or go, we'll check the slots for all of the sessions that could
 	 * have been active when we started our check.
 	 */
+	WT_STAT_FAST_CONN_INCR(session, cache_hazard_checks);
 	WT_ORDERED_READ(session_cnt, conn->session_cnt);
-	for (s = conn->sessions, i = 0; i < session_cnt; ++s, ++i) {
+	for (s = conn->sessions, i = 0, j = 0, max = 0;
+	    i < session_cnt; ++s, ++i) {
 		if (!s->active)
 			continue;
 		WT_ORDERED_READ(hazard_size, s->hazard_size);
-		for (hp = s->hazard; hp < s->hazard + hazard_size; ++hp)
-			if (hp->page == page)
+		if (s->hazard_size > max) {
+			max = s->hazard_size;
+			WT_STAT_FAST_CONN_SET(session,
+			    cache_hazard_max, max);
+		}
+		for (hp = s->hazard; hp < s->hazard + hazard_size; ++hp) {
+			++j;
+			if (hp->page == page) {
+				WT_STAT_FAST_CONN_INCRV(session,
+				    cache_hazard_walks, j);
 				return (hp);
+			}
+		}
 	}
+	WT_STAT_FAST_CONN_INCRV(session, cache_hazard_walks, j);
 	return (NULL);
 }
 
diff --git a/src/include/session.h b/src/include/session.h
index 7fdb7fc2548..aa51dae58c4 100644
--- a/src/include/session.h
+++ b/src/include/session.h
@@ -198,7 +198,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl {
 	((s)->hazard == NULL)
 
 	/* The number of hazard pointers grows dynamically. */
-#define	WT_HAZARD_INCR		10
+#define	WT_HAZARD_INCR		1
 	uint32_t   hazard_size;		/* Allocated slots in hazard array. */
 	uint32_t   nhazard;		/* Count of active hazard pointers */
 	WT_HAZARD *hazard;		/* Hazard pointer array */
diff --git a/src/include/stat.h b/src/include/stat.h
index 18461b1ee38..a71e0fa208e 100644
--- a/src/include/stat.h
+++ b/src/include/stat.h
@@ -275,6 +275,9 @@ struct __wt_connection_stats {
 	int64_t cache_eviction_worker_evicting;
 	int64_t cache_eviction_force_fail;
 	int64_t cache_eviction_hazard;
+	int64_t cache_hazard_checks;
+	int64_t cache_hazard_walks;
+	int64_t cache_hazard_max;
 	int64_t cache_inmem_splittable;
 	int64_t cache_inmem_split;
 	int64_t cache_eviction_internal;
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index 007df44f257..4aef84f5f8c 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -4303,257 +4303,263 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
 #define	WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL		1046
 /*! cache: hazard pointer blocked page eviction */
 #define	WT_STAT_CONN_CACHE_EVICTION_HAZARD		1047
+/*! cache: hazard pointer check calls */
+#define	WT_STAT_CONN_CACHE_HAZARD_CHECKS		1048
+/*! cache: hazard pointer check entries walked */
+#define	WT_STAT_CONN_CACHE_HAZARD_WALKS			1049
+/*! cache: hazard pointer maximum array length */
+#define	WT_STAT_CONN_CACHE_HAZARD_MAX			1050
 /*! cache: in-memory page passed criteria to be split */
-#define	WT_STAT_CONN_CACHE_INMEM_SPLITTABLE		1048
+#define	WT_STAT_CONN_CACHE_INMEM_SPLITTABLE		1051
 /*! cache: in-memory page splits */
-#define	WT_STAT_CONN_CACHE_INMEM_SPLIT			1049
+#define	WT_STAT_CONN_CACHE_INMEM_SPLIT			1052
 /*! cache: internal pages evicted */
-#define	WT_STAT_CONN_CACHE_EVICTION_INTERNAL		1050
+#define	WT_STAT_CONN_CACHE_EVICTION_INTERNAL		1053
 /*! cache: internal pages split during eviction */
-#define	WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL	1051
+#define	WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL	1054
 /*! cache: leaf pages split during eviction */
-#define	WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF		1052
+#define	WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF		1055
 /*! cache: lookaside table insert calls */
-#define	WT_STAT_CONN_CACHE_LOOKASIDE_INSERT		1053
+#define	WT_STAT_CONN_CACHE_LOOKASIDE_INSERT		1056
 /*! cache: lookaside table remove calls */
-#define	WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE		1054
+#define	WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE		1057
 /*! cache: maximum bytes configured */
-#define	WT_STAT_CONN_CACHE_BYTES_MAX			1055
+#define	WT_STAT_CONN_CACHE_BYTES_MAX			1058
 /*! cache: maximum page size at eviction */
-#define	WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE	1056
+#define	WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE	1059
 /*! cache: modified pages evicted */
-#define	WT_STAT_CONN_CACHE_EVICTION_DIRTY		1057
+#define	WT_STAT_CONN_CACHE_EVICTION_DIRTY		1060
 /*! cache: page split during eviction deepened the tree */
-#define	WT_STAT_CONN_CACHE_EVICTION_DEEPEN		1058
+#define	WT_STAT_CONN_CACHE_EVICTION_DEEPEN		1061
 /*! cache: page written requiring lookaside records */
-#define	WT_STAT_CONN_CACHE_WRITE_LOOKASIDE		1059
+#define	WT_STAT_CONN_CACHE_WRITE_LOOKASIDE		1062
 /*! cache: pages currently held in the cache */
-#define	WT_STAT_CONN_CACHE_PAGES_INUSE			1060
+#define	WT_STAT_CONN_CACHE_PAGES_INUSE			1063
 /*! cache: pages evicted because they exceeded the in-memory maximum */
-#define	WT_STAT_CONN_CACHE_EVICTION_FORCE		1061
+#define	WT_STAT_CONN_CACHE_EVICTION_FORCE		1064
 /*! cache: pages evicted because they had chains of deleted items */
-#define	WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE	1062
+#define	WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE	1065
 /*! cache: pages evicted by application threads */
-#define	WT_STAT_CONN_CACHE_EVICTION_APP			1063
+#define	WT_STAT_CONN_CACHE_EVICTION_APP			1066
 /*! cache: pages read into cache */
-#define	WT_STAT_CONN_CACHE_READ				1064
+#define	WT_STAT_CONN_CACHE_READ				1067
 /*! cache: pages read into cache requiring lookaside entries */
-#define	WT_STAT_CONN_CACHE_READ_LOOKASIDE		1065
+#define	WT_STAT_CONN_CACHE_READ_LOOKASIDE		1068
 /*! cache: pages requested from the cache */
-#define	WT_STAT_CONN_CACHE_PAGES_REQUESTED		1066
+#define	WT_STAT_CONN_CACHE_PAGES_REQUESTED		1069
 /*! cache: pages selected for eviction unable to be evicted */
-#define	WT_STAT_CONN_CACHE_EVICTION_FAIL		1067
+#define	WT_STAT_CONN_CACHE_EVICTION_FAIL		1070
 /*! cache: pages walked for eviction */
-#define	WT_STAT_CONN_CACHE_EVICTION_WALK		1068
+#define	WT_STAT_CONN_CACHE_EVICTION_WALK		1071
 /*! cache: pages written from cache */
-#define	WT_STAT_CONN_CACHE_WRITE			1069
+#define	WT_STAT_CONN_CACHE_WRITE			1072
 /*! cache: pages written requiring in-memory restoration */
-#define	WT_STAT_CONN_CACHE_WRITE_RESTORE		1070
+#define	WT_STAT_CONN_CACHE_WRITE_RESTORE		1073
 /*! cache: percentage overhead */
-#define	WT_STAT_CONN_CACHE_OVERHEAD			1071
+#define	WT_STAT_CONN_CACHE_OVERHEAD			1074
 /*! cache: tracked bytes belonging to internal pages in the cache */
-#define	WT_STAT_CONN_CACHE_BYTES_INTERNAL		1072
+#define	WT_STAT_CONN_CACHE_BYTES_INTERNAL		1075
 /*! cache: tracked bytes belonging to leaf pages in the cache */
-#define	WT_STAT_CONN_CACHE_BYTES_LEAF			1073
+#define	WT_STAT_CONN_CACHE_BYTES_LEAF			1076
 /*! cache: tracked bytes belonging to overflow pages in the cache */
-#define	WT_STAT_CONN_CACHE_BYTES_OVERFLOW		1074
+#define	WT_STAT_CONN_CACHE_BYTES_OVERFLOW		1077
 /*! cache: tracked dirty bytes in the cache */
-#define	WT_STAT_CONN_CACHE_BYTES_DIRTY			1075
+#define	WT_STAT_CONN_CACHE_BYTES_DIRTY			1078
 /*! cache: tracked dirty pages in the cache */
-#define	WT_STAT_CONN_CACHE_PAGES_DIRTY			1076
+#define	WT_STAT_CONN_CACHE_PAGES_DIRTY			1079
 /*! cache: unmodified pages evicted */
-#define	WT_STAT_CONN_CACHE_EVICTION_CLEAN		1077
+#define	WT_STAT_CONN_CACHE_EVICTION_CLEAN		1080
 /*! connection: auto adjusting condition resets */
-#define	WT_STAT_CONN_COND_AUTO_WAIT_RESET		1078
+#define	WT_STAT_CONN_COND_AUTO_WAIT_RESET		1081
 /*! connection: auto adjusting condition wait calls */
-#define	WT_STAT_CONN_COND_AUTO_WAIT			1079
+#define	WT_STAT_CONN_COND_AUTO_WAIT			1082
 /*! connection: files currently open */
-#define	WT_STAT_CONN_FILE_OPEN				1080
+#define	WT_STAT_CONN_FILE_OPEN				1083
 /*! connection: memory allocations */
-#define	WT_STAT_CONN_MEMORY_ALLOCATION			1081
+#define	WT_STAT_CONN_MEMORY_ALLOCATION			1084
 /*! connection: memory frees */
-#define	WT_STAT_CONN_MEMORY_FREE			1082
+#define	WT_STAT_CONN_MEMORY_FREE			1085
 /*! connection: memory re-allocations */
-#define	WT_STAT_CONN_MEMORY_GROW			1083
+#define	WT_STAT_CONN_MEMORY_GROW			1086
 /*! connection: pthread mutex condition wait calls */
-#define	WT_STAT_CONN_COND_WAIT				1084
+#define	WT_STAT_CONN_COND_WAIT				1087
 /*! connection: pthread mutex shared lock read-lock calls */
-#define	WT_STAT_CONN_RWLOCK_READ			1085
+#define	WT_STAT_CONN_RWLOCK_READ			1088
 /*! connection: pthread mutex shared lock write-lock calls */
-#define	WT_STAT_CONN_RWLOCK_WRITE			1086
+#define	WT_STAT_CONN_RWLOCK_WRITE			1089
 /*! connection: total read I/Os */
-#define	WT_STAT_CONN_READ_IO				1087
+#define	WT_STAT_CONN_READ_IO				1090
 /*! connection: total write I/Os */
-#define	WT_STAT_CONN_WRITE_IO				1088
+#define	WT_STAT_CONN_WRITE_IO				1091
 /*! cursor: cursor create calls */
-#define	WT_STAT_CONN_CURSOR_CREATE			1089
+#define	WT_STAT_CONN_CURSOR_CREATE			1092
 /*! cursor: cursor insert calls */
-#define	WT_STAT_CONN_CURSOR_INSERT			1090
+#define	WT_STAT_CONN_CURSOR_INSERT			1093
 /*! cursor: cursor next calls */
-#define	WT_STAT_CONN_CURSOR_NEXT			1091
+#define	WT_STAT_CONN_CURSOR_NEXT			1094
 /*! cursor: cursor prev calls */
-#define	WT_STAT_CONN_CURSOR_PREV			1092
+#define	WT_STAT_CONN_CURSOR_PREV			1095
 /*! cursor: cursor remove calls */
-#define	WT_STAT_CONN_CURSOR_REMOVE			1093
+#define	WT_STAT_CONN_CURSOR_REMOVE			1096
 /*! cursor: cursor reset calls */
-#define	WT_STAT_CONN_CURSOR_RESET			1094
+#define	WT_STAT_CONN_CURSOR_RESET			1097
 /*! cursor: cursor restarted searches */
-#define	WT_STAT_CONN_CURSOR_RESTART			1095
+#define	WT_STAT_CONN_CURSOR_RESTART			1098
 /*! cursor: cursor search calls */
-#define	WT_STAT_CONN_CURSOR_SEARCH			1096
+#define	WT_STAT_CONN_CURSOR_SEARCH			1099
 /*! cursor: cursor search near calls */
-#define	WT_STAT_CONN_CURSOR_SEARCH_NEAR			1097
+#define	WT_STAT_CONN_CURSOR_SEARCH_NEAR			1100
 /*! cursor: cursor update calls */
-#define	WT_STAT_CONN_CURSOR_UPDATE			1098
+#define	WT_STAT_CONN_CURSOR_UPDATE			1101
 /*! cursor: truncate calls */
-#define	WT_STAT_CONN_CURSOR_TRUNCATE			1099
+#define	WT_STAT_CONN_CURSOR_TRUNCATE			1102
 /*! data-handle: connection data handles currently active */
-#define	WT_STAT_CONN_DH_CONN_HANDLE_COUNT		1100
+#define	WT_STAT_CONN_DH_CONN_HANDLE_COUNT		1103
 /*! data-handle: connection sweep candidate became referenced */
-#define	WT_STAT_CONN_DH_SWEEP_REF			1101
+#define	WT_STAT_CONN_DH_SWEEP_REF			1104
 /*! data-handle: connection sweep dhandles closed */
-#define	WT_STAT_CONN_DH_SWEEP_CLOSE			1102
+#define	WT_STAT_CONN_DH_SWEEP_CLOSE			1105
 /*! data-handle: connection sweep dhandles removed from hash list */
-#define	WT_STAT_CONN_DH_SWEEP_REMOVE			1103
+#define	WT_STAT_CONN_DH_SWEEP_REMOVE			1106
 /*! data-handle: connection sweep time-of-death sets */
-#define	WT_STAT_CONN_DH_SWEEP_TOD			1104
+#define	WT_STAT_CONN_DH_SWEEP_TOD			1107
 /*! data-handle: connection sweeps */
-#define	WT_STAT_CONN_DH_SWEEPS				1105
+#define	WT_STAT_CONN_DH_SWEEPS				1108
 /*! data-handle: session dhandles swept */
-#define	WT_STAT_CONN_DH_SESSION_HANDLES			1106
+#define	WT_STAT_CONN_DH_SESSION_HANDLES			1109
 /*! data-handle: session sweep attempts */
-#define	WT_STAT_CONN_DH_SESSION_SWEEPS			1107
+#define	WT_STAT_CONN_DH_SESSION_SWEEPS			1110
 /*! log: busy returns attempting to switch slots */
-#define	WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY		1108
+#define	WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY		1111
 /*! log: consolidated slot closures */
-#define	WT_STAT_CONN_LOG_SLOT_CLOSES			1109
+#define	WT_STAT_CONN_LOG_SLOT_CLOSES			1112
 /*! log: consolidated slot join races */
-#define	WT_STAT_CONN_LOG_SLOT_RACES			1110
+#define	WT_STAT_CONN_LOG_SLOT_RACES			1113
 /*! log: consolidated slot join transitions */
-#define	WT_STAT_CONN_LOG_SLOT_TRANSITIONS		1111
+#define	WT_STAT_CONN_LOG_SLOT_TRANSITIONS		1114
 /*! log: consolidated slot joins */
-#define	WT_STAT_CONN_LOG_SLOT_JOINS			1112
+#define	WT_STAT_CONN_LOG_SLOT_JOINS			1115
 /*! log: consolidated slot unbuffered writes */
-#define	WT_STAT_CONN_LOG_SLOT_UNBUFFERED		1113
+#define	WT_STAT_CONN_LOG_SLOT_UNBUFFERED		1116
 /*! log: log bytes of payload data */
-#define	WT_STAT_CONN_LOG_BYTES_PAYLOAD			1114
+#define	WT_STAT_CONN_LOG_BYTES_PAYLOAD			1117
 /*! log: log bytes written */
-#define	WT_STAT_CONN_LOG_BYTES_WRITTEN			1115
+#define	WT_STAT_CONN_LOG_BYTES_WRITTEN			1118
 /*! log: log files manually zero-filled */
-#define	WT_STAT_CONN_LOG_ZERO_FILLS			1116
+#define	WT_STAT_CONN_LOG_ZERO_FILLS			1119
 /*! log: log flush operations */
-#define	WT_STAT_CONN_LOG_FLUSH				1117
+#define	WT_STAT_CONN_LOG_FLUSH				1120
 /*! log: log force write operations */
-#define	WT_STAT_CONN_LOG_FORCE_WRITE			1118
+#define	WT_STAT_CONN_LOG_FORCE_WRITE			1121
 /*! log: log force write operations skipped */
-#define	WT_STAT_CONN_LOG_FORCE_WRITE_SKIP		1119
+#define	WT_STAT_CONN_LOG_FORCE_WRITE_SKIP		1122
 /*! log: log records compressed */
-#define	WT_STAT_CONN_LOG_COMPRESS_WRITES		1120
+#define	WT_STAT_CONN_LOG_COMPRESS_WRITES		1123
 /*! log: log records not compressed */
-#define	WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS		1121
+#define	WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS		1124
 /*! log: log records too small to compress */
-#define	WT_STAT_CONN_LOG_COMPRESS_SMALL			1122
+#define	WT_STAT_CONN_LOG_COMPRESS_SMALL			1125
 /*! log: log release advances write LSN */
-#define	WT_STAT_CONN_LOG_RELEASE_WRITE_LSN		1123
+#define	WT_STAT_CONN_LOG_RELEASE_WRITE_LSN		1126
 /*! log: log scan operations */
-#define	WT_STAT_CONN_LOG_SCANS				1124
+#define	WT_STAT_CONN_LOG_SCANS				1127
 /*! log: log scan records requiring two reads */
-#define	WT_STAT_CONN_LOG_SCAN_REREADS			1125
+#define	WT_STAT_CONN_LOG_SCAN_REREADS			1128
 /*! log: log server thread advances write LSN */
-#define	WT_STAT_CONN_LOG_WRITE_LSN			1126
+#define	WT_STAT_CONN_LOG_WRITE_LSN			1129
 /*! log: log server thread write LSN walk skipped */
-#define	WT_STAT_CONN_LOG_WRITE_LSN_SKIP			1127
+#define	WT_STAT_CONN_LOG_WRITE_LSN_SKIP			1130
 /*! log: log sync operations */
-#define	WT_STAT_CONN_LOG_SYNC				1128
+#define	WT_STAT_CONN_LOG_SYNC				1131
 /*! log: log sync_dir operations */
-#define	WT_STAT_CONN_LOG_SYNC_DIR			1129
+#define	WT_STAT_CONN_LOG_SYNC_DIR			1132
 /*! log: log write operations */
-#define	WT_STAT_CONN_LOG_WRITES				1130
+#define	WT_STAT_CONN_LOG_WRITES				1133
 /*! log: logging bytes consolidated */
-#define	WT_STAT_CONN_LOG_SLOT_CONSOLIDATED		1131
+#define	WT_STAT_CONN_LOG_SLOT_CONSOLIDATED		1134
 /*! log: maximum log file size */
-#define	WT_STAT_CONN_LOG_MAX_FILESIZE			1132
+#define	WT_STAT_CONN_LOG_MAX_FILESIZE			1135
 /*! log: number of pre-allocated log files to create */
-#define	WT_STAT_CONN_LOG_PREALLOC_MAX			1133
+#define	WT_STAT_CONN_LOG_PREALLOC_MAX			1136
 /*! log: pre-allocated log files not ready and missed */
-#define	WT_STAT_CONN_LOG_PREALLOC_MISSED		1134
+#define	WT_STAT_CONN_LOG_PREALLOC_MISSED		1137
 /*! log: pre-allocated log files prepared */
-#define	WT_STAT_CONN_LOG_PREALLOC_FILES			1135
+#define	WT_STAT_CONN_LOG_PREALLOC_FILES			1138
 /*! log: pre-allocated log files used */
-#define	WT_STAT_CONN_LOG_PREALLOC_USED			1136
+#define	WT_STAT_CONN_LOG_PREALLOC_USED			1139
 /*! log: records processed by log scan */
-#define	WT_STAT_CONN_LOG_SCAN_RECORDS			1137
+#define	WT_STAT_CONN_LOG_SCAN_RECORDS			1140
 /*! log: total in-memory size of compressed records */
-#define	WT_STAT_CONN_LOG_COMPRESS_MEM			1138
+#define	WT_STAT_CONN_LOG_COMPRESS_MEM			1141
 /*! log: total log buffer size */
-#define	WT_STAT_CONN_LOG_BUFFER_SIZE			1139
+#define	WT_STAT_CONN_LOG_BUFFER_SIZE			1142
 /*! log: total size of compressed records */
-#define	WT_STAT_CONN_LOG_COMPRESS_LEN			1140
+#define	WT_STAT_CONN_LOG_COMPRESS_LEN			1143
 /*! log: written slots coalesced */
-#define	WT_STAT_CONN_LOG_SLOT_COALESCED			1141
+#define	WT_STAT_CONN_LOG_SLOT_COALESCED			1144
 /*! log: yields waiting for previous log file close */
-#define	WT_STAT_CONN_LOG_CLOSE_YIELDS			1142
+#define	WT_STAT_CONN_LOG_CLOSE_YIELDS			1145
 /*! reconciliation: fast-path pages deleted */
-#define	WT_STAT_CONN_REC_PAGE_DELETE_FAST		1143
+#define	WT_STAT_CONN_REC_PAGE_DELETE_FAST		1146
 /*! reconciliation: page reconciliation calls */
-#define	WT_STAT_CONN_REC_PAGES				1144
+#define	WT_STAT_CONN_REC_PAGES				1147
 /*! reconciliation: page reconciliation calls for eviction */
-#define	WT_STAT_CONN_REC_PAGES_EVICTION			1145
+#define	WT_STAT_CONN_REC_PAGES_EVICTION			1148
 /*! reconciliation: pages deleted */
-#define	WT_STAT_CONN_REC_PAGE_DELETE			1146
+#define	WT_STAT_CONN_REC_PAGE_DELETE			1149
 /*! reconciliation: split bytes currently awaiting free */
-#define	WT_STAT_CONN_REC_SPLIT_STASHED_BYTES		1147
+#define	WT_STAT_CONN_REC_SPLIT_STASHED_BYTES		1150
 /*! reconciliation: split objects currently awaiting free */
-#define	WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS		1148
+#define	WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS		1151
 /*! session: open cursor count */
-#define	WT_STAT_CONN_SESSION_CURSOR_OPEN		1149
+#define	WT_STAT_CONN_SESSION_CURSOR_OPEN		1152
 /*! session: open session count */
-#define	WT_STAT_CONN_SESSION_OPEN			1150
+#define	WT_STAT_CONN_SESSION_OPEN			1153
 /*! thread-yield: page acquire busy blocked */
-#define	WT_STAT_CONN_PAGE_BUSY_BLOCKED			1151
+#define	WT_STAT_CONN_PAGE_BUSY_BLOCKED			1154
 /*! thread-yield: page acquire eviction blocked */
-#define	WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED	1152
+#define	WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED	1155
 /*! thread-yield: page acquire locked blocked */
-#define	WT_STAT_CONN_PAGE_LOCKED_BLOCKED		1153
+#define	WT_STAT_CONN_PAGE_LOCKED_BLOCKED		1156
 /*! thread-yield: page acquire read blocked */
-#define	WT_STAT_CONN_PAGE_READ_BLOCKED			1154
+#define	WT_STAT_CONN_PAGE_READ_BLOCKED			1157
 /*! thread-yield: page acquire time sleeping (usecs) */
-#define	WT_STAT_CONN_PAGE_SLEEP				1155
+#define	WT_STAT_CONN_PAGE_SLEEP				1158
 /*! transaction: number of named snapshots created */
-#define	WT_STAT_CONN_TXN_SNAPSHOTS_CREATED		1156
+#define	WT_STAT_CONN_TXN_SNAPSHOTS_CREATED		1159
 /*! transaction: number of named snapshots dropped */
-#define	WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED		1157
+#define	WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED		1160
 /*! transaction: transaction begins */
-#define	WT_STAT_CONN_TXN_BEGIN				1158
+#define	WT_STAT_CONN_TXN_BEGIN				1161
 /*! transaction: transaction checkpoint currently running */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_RUNNING		1159
+#define	WT_STAT_CONN_TXN_CHECKPOINT_RUNNING		1162
 /*! transaction: transaction checkpoint generation */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_GENERATION		1160
+#define	WT_STAT_CONN_TXN_CHECKPOINT_GENERATION		1163
 /*! transaction: transaction checkpoint max time (msecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX		1161
+#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX		1164
 /*! transaction: transaction checkpoint min time (msecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN		1162
+#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN		1165
 /*! transaction: transaction checkpoint most recent time (msecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT		1163
+#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT		1166
 /*! transaction: transaction checkpoint total time (msecs) */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL		1164
+#define	WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL		1167
 /*! transaction: transaction checkpoints */
-#define	WT_STAT_CONN_TXN_CHECKPOINT			1165
+#define	WT_STAT_CONN_TXN_CHECKPOINT			1168
 /*! transaction: transaction failures due to cache overflow */
-#define	WT_STAT_CONN_TXN_FAIL_CACHE			1166
+#define	WT_STAT_CONN_TXN_FAIL_CACHE			1169
 /*! transaction: transaction range of IDs currently pinned */
-#define	WT_STAT_CONN_TXN_PINNED_RANGE			1167
+#define	WT_STAT_CONN_TXN_PINNED_RANGE			1170
 /*! transaction: transaction range of IDs currently pinned by a checkpoint */
-#define	WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE	1168
+#define	WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE	1171
 /*! transaction: transaction range of IDs currently pinned by named
  * snapshots */
-#define	WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE		1169
+#define	WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE		1172
 /*! transaction: transaction sync calls */
-#define	WT_STAT_CONN_TXN_SYNC				1170
+#define	WT_STAT_CONN_TXN_SYNC				1173
 /*! transaction: transactions committed */
-#define	WT_STAT_CONN_TXN_COMMIT				1171
+#define	WT_STAT_CONN_TXN_COMMIT				1174
 /*! transaction: transactions rolled back */
-#define	WT_STAT_CONN_TXN_ROLLBACK			1172
+#define	WT_STAT_CONN_TXN_ROLLBACK			1175
 
 /*!
  * @}
diff --git a/src/session/session_api.c b/src/session/session_api.c
index 933f2273902..8740c983110 100644
--- a/src/session/session_api.c
+++ b/src/session/session_api.c
@@ -1675,7 +1675,7 @@ __open_session(WT_CONNECTION_IMPL *conn,
 	 * __wt_hazard_close ensures the array is cleared - so it is safe to
 	 * reset the starting size on each open.
 	 */
-	session_ret->hazard_size = WT_HAZARD_INCR;
+	session_ret->hazard_size = 0;
 
 	/*
 	 * Configuration: currently, the configuration for open_session is the
diff --git a/src/support/stat.c b/src/support/stat.c
index bb46ad03e43..7514aac56c4 100644
--- a/src/support/stat.c
+++ b/src/support/stat.c
@@ -562,6 +562,9 @@ static const char * const __stats_connection_desc[] = {
 	"cache: eviction worker thread evicting pages",
 	"cache: failed eviction of pages that exceeded the in-memory maximum",
 	"cache: hazard pointer blocked page eviction",
+	"cache: hazard pointer check calls",
+	"cache: hazard pointer check entries walked",
+	"cache: hazard pointer maximum array length",
 	"cache: in-memory page passed criteria to be split",
 	"cache: in-memory page splits",
 	"cache: internal pages evicted",
@@ -765,6 +768,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats)
 	stats->cache_eviction_worker_evicting = 0;
 	stats->cache_eviction_force_fail = 0;
 	stats->cache_eviction_hazard = 0;
+	stats->cache_hazard_checks = 0;
+	stats->cache_hazard_walks = 0;
+	stats->cache_hazard_max = 0;
 	stats->cache_inmem_splittable = 0;
 	stats->cache_inmem_split = 0;
 	stats->cache_eviction_internal = 0;
@@ -905,6 +911,8 @@ void
 __wt_stat_connection_aggregate(
     WT_CONNECTION_STATS **from, WT_CONNECTION_STATS *to)
 {
+	int64_t v;
+
 	to->lsm_work_queue_app += WT_STAT_READ(from, lsm_work_queue_app);
 	to->lsm_work_queue_manager +=
 	    WT_STAT_READ(from, lsm_work_queue_manager);
@@ -972,6 +980,10 @@ __wt_stat_connection_aggregate(
 	    WT_STAT_READ(from, cache_eviction_force_fail);
 	to->cache_eviction_hazard +=
 	    WT_STAT_READ(from, cache_eviction_hazard);
+	to->cache_hazard_checks += WT_STAT_READ(from, cache_hazard_checks);
+	to->cache_hazard_walks += WT_STAT_READ(from, cache_hazard_walks);
+	if ((v = WT_STAT_READ(from, cache_hazard_max)) > to->cache_hazard_max)
+		to->cache_hazard_max = v;
 	to->cache_inmem_splittable +=
 	    WT_STAT_READ(from, cache_inmem_splittable);
 	to->cache_inmem_split += WT_STAT_READ(from, cache_inmem_split);
diff --git a/tools/wtstats/stat_data.py b/tools/wtstats/stat_data.py
index c75e4f194dd..eca3c137738 100644
--- a/tools/wtstats/stat_data.py
+++ b/tools/wtstats/stat_data.py
@@ -5,6 +5,7 @@ no_scale_per_second_list = [
     'async: maximum work queue length',
     'cache: bytes currently in the cache',
     'cache: eviction currently operating in aggressive mode',
+    'cache: hazard pointer maximum array length',
     'cache: maximum bytes configured',
     'cache: maximum page size at eviction',
     'cache: pages currently held in the cache',