Add backoff to sleep if no async work available. #1002

6 files changed, 128 insertions, 98 deletions

diff --git a/dist/stat_data.py b/dist/stat_data.py
index ae5cdfd9ebd..1244a1a3700 100644
--- a/dist/stat_data.py
+++ b/dist/stat_data.py
@@ -50,6 +50,7 @@ connection_stats = [
 	Stat('async_cur_queue', 'async: current work queue length'),
 	Stat('async_max_queue', 'async: maximum work queue length',
 	    'max_aggregate,no_scale'),
+	Stat('async_nowork', 'async: number of times worker found no work'),
 	Stat('async_op_alloc', 'async op allocation'),
 	Stat('async_op_compact', 'async op compact calls'),
 	Stat('async_op_insert', 'async op insert calls'),
diff --git a/src/async/async_worker.c b/src/async/async_worker.c
index 42a87014de7..812d695d527 100644
--- a/src/async/async_worker.c
+++ b/src/async/async_worker.c
@@ -17,28 +17,48 @@ __async_op_dequeue(WT_CONNECTION_IMPL *conn, WT_SESSION_IMPL *session,
     WT_ASYNC_OP_IMPL **op)
 {
 	WT_ASYNC *async;
+	long sleep_usec;
 	uint64_t cur_tail, last_consume, my_consume, my_slot, prev_slot;
-	uint32_t max_tries, tries;
+	uint32_t tries;
 
 	async = conn->async;
 	*op = NULL;
-	max_tries = 100;
-	tries = 0;
 	/*
 	 * Wait for work to do.  Work is available when async->head moves.
 	 * Then grab the slot containing the work.  If we lose, try again.
 	 */
 retry:
+	tries = 0;
+	sleep_usec = 100;
 	WT_ORDERED_READ(last_consume, async->alloc_tail);
-	while (last_consume == async->head && ++tries < max_tries &&
+	/*
+	 * We stay in this loop until there is work to do.
+	 */
+	while (last_consume == async->head &&
 	    async->flush_state != WT_ASYNC_FLUSHING) {
-		__wt_yield();
+		WT_STAT_FAST_CONN_INCR(session, async_nowork);
+		if (++tries < MAX_ASYNC_YIELD)
+			/*
+			 * Initially when we find no work, allow other
+			 * threads to run.
+			 */
+			__wt_yield();
+		else {
+			/*
+			 * If we haven't found work in a while, start sleeping
+			 * to wait for work to arrive instead of spinning.
+			 */
+			__wt_sleep(0, sleep_usec);
+			sleep_usec = WT_MIN(sleep_usec * 2,
+			    MAX_ASYNC_SLEEP_USECS);
+		}
+		if (!F_ISSET(conn, WT_CONN_SERVER_RUN))
+			return (0);
+		if (F_ISSET(conn, WT_CONN_PANIC))
+			return (__wt_panic(session));
 		WT_ORDERED_READ(last_consume, async->alloc_tail);
 	}
-	if (F_ISSET(conn, WT_CONN_PANIC))
-		return (__wt_panic(session));
-	if (tries >= max_tries ||
-	    async->flush_state == WT_ASYNC_FLUSHING)
+	if (async->flush_state == WT_ASYNC_FLUSHING)
 		return (0);
 	/*
 	 * Try to increment the tail to claim this slot.  If we lose
diff --git a/src/include/async.h b/src/include/async.h
index 27a645e68a8..293dc5c9b7f 100644
--- a/src/include/async.h
+++ b/src/include/async.h
@@ -19,6 +19,9 @@ typedef enum {
 	WT_ASYNC_FLUSHING		/* Notify workers */
 } WT_ASYNC_FLUSH_STATE;
 
+#define	MAX_ASYNC_SLEEP_USECS	100000	/* Maximum sleep waiting for work */
+#define	MAX_ASYNC_YIELD		200	/* Maximum number of yields for work */
+
 #define	O2C(op)	((WT_CONNECTION_IMPL *)(op)->iface.connection)
 #define	O2S(op)								\
     (((WT_CONNECTION_IMPL *)(op)->iface.connection)->default_session)
diff --git a/src/include/stat.h b/src/include/stat.h
index 572550d9b6b..f79b4133f57 100644
--- a/src/include/stat.h
+++ b/src/include/stat.h
@@ -123,6 +123,7 @@ struct __wt_connection_stats {
 	WT_STATS async_flush;
 	WT_STATS async_full;
 	WT_STATS async_max_queue;
+	WT_STATS async_nowork;
 	WT_STATS async_op_alloc;
 	WT_STATS async_op_compact;
 	WT_STATS async_op_insert;
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index b8fa8bffee9..385be1ab49b 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -3032,184 +3032,186 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
 #define	WT_STAT_CONN_ASYNC_FULL				1004
 /*! async: maximum work queue length */
 #define	WT_STAT_CONN_ASYNC_MAX_QUEUE			1005
+/*! async: number of times worker found no work */
+#define	WT_STAT_CONN_ASYNC_NOWORK			1006
 /*! async op allocation */
-#define	WT_STAT_CONN_ASYNC_OP_ALLOC			1006
+#define	WT_STAT_CONN_ASYNC_OP_ALLOC			1007
 /*! async op compact calls */
-#define	WT_STAT_CONN_ASYNC_OP_COMPACT			1007
+#define	WT_STAT_CONN_ASYNC_OP_COMPACT			1008
 /*! async op insert calls */
-#define	WT_STAT_CONN_ASYNC_OP_INSERT			1008
+#define	WT_STAT_CONN_ASYNC_OP_INSERT			1009
 /*! async op remove calls */
-#define	WT_STAT_CONN_ASYNC_OP_REMOVE			1009
+#define	WT_STAT_CONN_ASYNC_OP_REMOVE			1010
 /*! async op search calls */
-#define	WT_STAT_CONN_ASYNC_OP_SEARCH			1010
+#define	WT_STAT_CONN_ASYNC_OP_SEARCH			1011
 /*! async op update calls */
-#define	WT_STAT_CONN_ASYNC_OP_UPDATE			1011
+#define	WT_STAT_CONN_ASYNC_OP_UPDATE			1012
 /*! block manager: mapped bytes read */
-#define	WT_STAT_CONN_BLOCK_BYTE_MAP_READ		1012
+#define	WT_STAT_CONN_BLOCK_BYTE_MAP_READ		1013
 /*! block manager: bytes read */
-#define	WT_STAT_CONN_BLOCK_BYTE_READ			1013
+#define	WT_STAT_CONN_BLOCK_BYTE_READ			1014
 /*! block manager: bytes written */
-#define	WT_STAT_CONN_BLOCK_BYTE_WRITE			1014
+#define	WT_STAT_CONN_BLOCK_BYTE_WRITE			1015
 /*! block manager: mapped blocks read */
-#define	WT_STAT_CONN_BLOCK_MAP_READ			1015
+#define	WT_STAT_CONN_BLOCK_MAP_READ			1016
 /*! block manager: blocks pre-loaded */
-#define	WT_STAT_CONN_BLOCK_PRELOAD			1016
+#define	WT_STAT_CONN_BLOCK_PRELOAD			1017
 /*! block manager: blocks read */
-#define	WT_STAT_CONN_BLOCK_READ				1017
+#define	WT_STAT_CONN_BLOCK_READ				1018
 /*! block manager: blocks written */
-#define	WT_STAT_CONN_BLOCK_WRITE			1018
+#define	WT_STAT_CONN_BLOCK_WRITE			1019
 /*! cache: tracked dirty bytes in the cache */
-#define	WT_STAT_CONN_CACHE_BYTES_DIRTY			1019
+#define	WT_STAT_CONN_CACHE_BYTES_DIRTY			1020
 /*! cache: bytes currently in the cache */
-#define	WT_STAT_CONN_CACHE_BYTES_INUSE			1020
+#define	WT_STAT_CONN_CACHE_BYTES_INUSE			1021
 /*! cache: maximum bytes configured */
-#define	WT_STAT_CONN_CACHE_BYTES_MAX			1021
+#define	WT_STAT_CONN_CACHE_BYTES_MAX			1022
 /*! cache: bytes read into cache */
-#define	WT_STAT_CONN_CACHE_BYTES_READ			1022
+#define	WT_STAT_CONN_CACHE_BYTES_READ			1023
 /*! cache: bytes written from cache */
-#define	WT_STAT_CONN_CACHE_BYTES_WRITE			1023
+#define	WT_STAT_CONN_CACHE_BYTES_WRITE			1024
 /*! cache: checkpoint blocked page eviction */
-#define	WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT		1024
+#define	WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT		1025
 /*! cache: unmodified pages evicted */
-#define	WT_STAT_CONN_CACHE_EVICTION_CLEAN		1025
+#define	WT_STAT_CONN_CACHE_EVICTION_CLEAN		1026
 /*! cache: page split during eviction deepened the tree */
-#define	WT_STAT_CONN_CACHE_EVICTION_DEEPEN		1026
+#define	WT_STAT_CONN_CACHE_EVICTION_DEEPEN		1027
 /*! cache: modified pages evicted */
-#define	WT_STAT_CONN_CACHE_EVICTION_DIRTY		1027
+#define	WT_STAT_CONN_CACHE_EVICTION_DIRTY		1028
 /*! cache: pages selected for eviction unable to be evicted */
-#define	WT_STAT_CONN_CACHE_EVICTION_FAIL		1028
+#define	WT_STAT_CONN_CACHE_EVICTION_FAIL		1029
 /*! cache: pages evicted because they exceeded the in-memory maximum */
-#define	WT_STAT_CONN_CACHE_EVICTION_FORCE		1029
+#define	WT_STAT_CONN_CACHE_EVICTION_FORCE		1030
 /*! cache: failed eviction of pages that exceeded the in-memory maximum */
-#define	WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL		1030
+#define	WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL		1031
 /*! cache: hazard pointer blocked page eviction */
-#define	WT_STAT_CONN_CACHE_EVICTION_HAZARD		1031
+#define	WT_STAT_CONN_CACHE_EVICTION_HAZARD		1032
 /*! cache: internal pages evicted */
-#define	WT_STAT_CONN_CACHE_EVICTION_INTERNAL		1032
+#define	WT_STAT_CONN_CACHE_EVICTION_INTERNAL		1033
 /*! cache: eviction server unable to reach eviction goal */
-#define	WT_STAT_CONN_CACHE_EVICTION_SLOW		1033
+#define	WT_STAT_CONN_CACHE_EVICTION_SLOW		1034
 /*! cache: pages split during eviction */
-#define	WT_STAT_CONN_CACHE_EVICTION_SPLIT		1034
+#define	WT_STAT_CONN_CACHE_EVICTION_SPLIT		1035
 /*! cache: pages walked for eviction */
-#define	WT_STAT_CONN_CACHE_EVICTION_WALK		1035
+#define	WT_STAT_CONN_CACHE_EVICTION_WALK		1036
 /*! cache: tracked dirty pages in the cache */
-#define	WT_STAT_CONN_CACHE_PAGES_DIRTY			1036
+#define	WT_STAT_CONN_CACHE_PAGES_DIRTY			1037
 /*! cache: pages currently held in the cache */
-#define	WT_STAT_CONN_CACHE_PAGES_INUSE			1037
+#define	WT_STAT_CONN_CACHE_PAGES_INUSE			1038
 /*! cache: pages read into cache */
-#define	WT_STAT_CONN_CACHE_READ				1038
+#define	WT_STAT_CONN_CACHE_READ				1039
 /*! cache: pages written from cache */
-#define	WT_STAT_CONN_CACHE_WRITE			1039
+#define	WT_STAT_CONN_CACHE_WRITE			1040
 /*! pthread mutex condition wait calls */
-#define	WT_STAT_CONN_COND_WAIT				1040
+#define	WT_STAT_CONN_COND_WAIT				1041
 /*! cursor creation */
-#define	WT_STAT_CONN_CURSOR_CREATE			1041
+#define	WT_STAT_CONN_CURSOR_CREATE			1042
 /*! Btree cursor insert calls */
-#define	WT_STAT_CONN_CURSOR_INSERT			1042
+#define	WT_STAT_CONN_CURSOR_INSERT			1043
 /*! Btree cursor next calls */
-#define	WT_STAT_CONN_CURSOR_NEXT			1043
+#define	WT_STAT_CONN_CURSOR_NEXT			1044
 /*! Btree cursor prev calls */
-#define	WT_STAT_CONN_CURSOR_PREV			1044
+#define	WT_STAT_CONN_CURSOR_PREV			1045
 /*! Btree cursor remove calls */
-#define	WT_STAT_CONN_CURSOR_REMOVE			1045
+#define	WT_STAT_CONN_CURSOR_REMOVE			1046
 /*! Btree cursor reset calls */
-#define	WT_STAT_CONN_CURSOR_RESET			1046
+#define	WT_STAT_CONN_CURSOR_RESET			1047
 /*! Btree cursor search calls */
-#define	WT_STAT_CONN_CURSOR_SEARCH			1047
+#define	WT_STAT_CONN_CURSOR_SEARCH			1048
 /*! Btree cursor search near calls */
-#define	WT_STAT_CONN_CURSOR_SEARCH_NEAR			1048
+#define	WT_STAT_CONN_CURSOR_SEARCH_NEAR			1049
 /*! Btree cursor update calls */
-#define	WT_STAT_CONN_CURSOR_UPDATE			1049
+#define	WT_STAT_CONN_CURSOR_UPDATE			1050
 /*! dhandle: connection dhandles swept */
-#define	WT_STAT_CONN_DH_CONN_HANDLES			1050
+#define	WT_STAT_CONN_DH_CONN_HANDLES			1051
 /*! dhandle: connection sweep attempts */
-#define	WT_STAT_CONN_DH_CONN_SWEEPS			1051
+#define	WT_STAT_CONN_DH_CONN_SWEEPS			1052
 /*! dhandle: session dhandles swept */
-#define	WT_STAT_CONN_DH_SESSION_HANDLES			1052
+#define	WT_STAT_CONN_DH_SESSION_HANDLES			1053
 /*! dhandle: session sweep attempts */
-#define	WT_STAT_CONN_DH_SESSION_SWEEPS			1053
+#define	WT_STAT_CONN_DH_SESSION_SWEEPS			1054
 /*! dhandle: sweeps conflicting with evict */
-#define	WT_STAT_CONN_DH_SWEEP_EVICT			1054
+#define	WT_STAT_CONN_DH_SWEEP_EVICT			1055
 /*! files currently open */
-#define	WT_STAT_CONN_FILE_OPEN				1055
+#define	WT_STAT_CONN_FILE_OPEN				1056
 /*! log: log buffer size increases */
-#define	WT_STAT_CONN_LOG_BUFFER_GROW			1056
+#define	WT_STAT_CONN_LOG_BUFFER_GROW			1057
 /*! log: total log buffer size */
-#define	WT_STAT_CONN_LOG_BUFFER_SIZE			1057
+#define	WT_STAT_CONN_LOG_BUFFER_SIZE			1058
 /*! log: user provided log bytes written */
-#define	WT_STAT_CONN_LOG_BYTES_USER			1058
+#define	WT_STAT_CONN_LOG_BYTES_USER			1059
 /*! log: log bytes written */
-#define	WT_STAT_CONN_LOG_BYTES_WRITTEN			1059
+#define	WT_STAT_CONN_LOG_BYTES_WRITTEN			1060
 /*! log: maximum log file size */
-#define	WT_STAT_CONN_LOG_MAX_FILESIZE			1060
+#define	WT_STAT_CONN_LOG_MAX_FILESIZE			1061
 /*! log: log read operations */
-#define	WT_STAT_CONN_LOG_READS				1061
+#define	WT_STAT_CONN_LOG_READS				1062
 /*! log: records processed by log scan */
-#define	WT_STAT_CONN_LOG_SCAN_RECORDS			1062
+#define	WT_STAT_CONN_LOG_SCAN_RECORDS			1063
 /*! log: log scan records requiring two reads */
-#define	WT_STAT_CONN_LOG_SCAN_REREADS			1063
+#define	WT_STAT_CONN_LOG_SCAN_REREADS			1064
 /*! log: log scan operations */
-#define	WT_STAT_CONN_LOG_SCANS				1064
+#define	WT_STAT_CONN_LOG_SCANS				1065
 /*! log: consolidated slot closures */
-#define	WT_STAT_CONN_LOG_SLOT_CLOSES			1065
+#define	WT_STAT_CONN_LOG_SLOT_CLOSES			1066
 /*! log: logging bytes consolidated */
-#define	WT_STAT_CONN_LOG_SLOT_CONSOLIDATED		1066
+#define	WT_STAT_CONN_LOG_SLOT_CONSOLIDATED		1067
 /*! log: consolidated slot joins */
-#define	WT_STAT_CONN_LOG_SLOT_JOINS			1067
+#define	WT_STAT_CONN_LOG_SLOT_JOINS			1068
 /*! log: consolidated slot join races */
-#define	WT_STAT_CONN_LOG_SLOT_RACES			1068
+#define	WT_STAT_CONN_LOG_SLOT_RACES			1069
 /*! log: slots selected for switching that were unavailable */
-#define	WT_STAT_CONN_LOG_SLOT_SWITCH_FAILS		1069
+#define	WT_STAT_CONN_LOG_SLOT_SWITCH_FAILS		1070
 /*! log: record size exceeded maximum */
-#define	WT_STAT_CONN_LOG_SLOT_TOOBIG			1070
+#define	WT_STAT_CONN_LOG_SLOT_TOOBIG			1071
 /*! log: failed to find a slot large enough for record */
-#define	WT_STAT_CONN_LOG_SLOT_TOOSMALL			1071
+#define	WT_STAT_CONN_LOG_SLOT_TOOSMALL			1072
 /*! log: consolidated slot join transitions */
-#define	WT_STAT_CONN_LOG_SLOT_TRANSITIONS		1072
+#define	WT_STAT_CONN_LOG_SLOT_TRANSITIONS		1073
 /*! log: log sync operations */
-#define	WT_STAT_CONN_LOG_SYNC				1073
+#define	WT_STAT_CONN_LOG_SYNC				1074
 /*! log: log write operations */
-#define	WT_STAT_CONN_LOG_WRITES				1074
+#define	WT_STAT_CONN_LOG_WRITES				1075
 /*! sleep for LSM checkpoint throttle */
-#define	WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE		1075
+#define	WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE		1076
 /*! sleep for LSM merge throttle */
-#define	WT_STAT_CONN_LSM_MERGE_THROTTLE			1076
+#define	WT_STAT_CONN_LSM_MERGE_THROTTLE			1077
 /*! rows merged in an LSM tree */
-#define	WT_STAT_CONN_LSM_ROWS_MERGED			1077
+#define	WT_STAT_CONN_LSM_ROWS_MERGED			1078
 /*! memory allocations */
-#define	WT_STAT_CONN_MEMORY_ALLOCATION			1078
+#define	WT_STAT_CONN_MEMORY_ALLOCATION			1079
 /*! memory frees */
-#define	WT_STAT_CONN_MEMORY_FREE			1079
+#define	WT_STAT_CONN_MEMORY_FREE			1080
 /*! memory re-allocations */
-#define	WT_STAT_CONN_MEMORY_GROW			1080
+#define	WT_STAT_CONN_MEMORY_GROW			1081
 /*! total read I/Os */
-#define	WT_STAT_CONN_READ_IO				1081
+#define	WT_STAT_CONN_READ_IO				1082
 /*! page reconciliation calls */
-#define	WT_STAT_CONN_REC_PAGES				1082
+#define	WT_STAT_CONN_REC_PAGES				1083
 /*! page reconciliation calls for eviction */
-#define	WT_STAT_CONN_REC_PAGES_EVICTION			1083
+#define	WT_STAT_CONN_REC_PAGES_EVICTION			1084
 /*! reconciliation failed because an update could not be included */
-#define	WT_STAT_CONN_REC_SKIPPED_UPDATE			1084
+#define	WT_STAT_CONN_REC_SKIPPED_UPDATE			1085
 /*! pthread mutex shared lock read-lock calls */
-#define	WT_STAT_CONN_RWLOCK_READ			1085
+#define	WT_STAT_CONN_RWLOCK_READ			1086
 /*! pthread mutex shared lock write-lock calls */
-#define	WT_STAT_CONN_RWLOCK_WRITE			1086
+#define	WT_STAT_CONN_RWLOCK_WRITE			1087
 /*! open cursor count */
-#define	WT_STAT_CONN_SESSION_CURSOR_OPEN		1087
+#define	WT_STAT_CONN_SESSION_CURSOR_OPEN		1088
 /*! transactions */
-#define	WT_STAT_CONN_TXN_BEGIN				1088
+#define	WT_STAT_CONN_TXN_BEGIN				1089
 /*! transaction checkpoints */
-#define	WT_STAT_CONN_TXN_CHECKPOINT			1089
+#define	WT_STAT_CONN_TXN_CHECKPOINT			1090
 /*! transaction checkpoint currently running */
-#define	WT_STAT_CONN_TXN_CHECKPOINT_RUNNING		1090
+#define	WT_STAT_CONN_TXN_CHECKPOINT_RUNNING		1091
 /*! transactions committed */
-#define	WT_STAT_CONN_TXN_COMMIT				1091
+#define	WT_STAT_CONN_TXN_COMMIT				1092
 /*! transaction failures due to cache overflow */
-#define	WT_STAT_CONN_TXN_FAIL_CACHE			1092
+#define	WT_STAT_CONN_TXN_FAIL_CACHE			1093
 /*! transactions rolled-back */
-#define	WT_STAT_CONN_TXN_ROLLBACK			1093
+#define	WT_STAT_CONN_TXN_ROLLBACK			1094
 /*! total write I/Os */
-#define	WT_STAT_CONN_WRITE_IO				1094
+#define	WT_STAT_CONN_WRITE_IO				1095
 
 /*!
  * @}
diff --git a/src/support/stat.c b/src/support/stat.c
index c30420dc7e3..7e30b2b2214 100644
--- a/src/support/stat.c
+++ b/src/support/stat.c
@@ -317,6 +317,8 @@ __wt_stat_init_connection_stats(WT_CONNECTION_STATS *stats)
 	stats->async_flush.desc = "async: number of async flush calls";
 	stats->async_full.desc = "async: number of times op allocation failed";
 	stats->async_max_queue.desc = "async: maximum work queue length";
+	stats->async_nowork.desc =
+	    "async: number of times worker found no work";
 	stats->async_op_alloc.desc = "async op allocation";
 	stats->async_op_compact.desc = "async op compact calls";
 	stats->async_op_insert.desc = "async op insert calls";
@@ -441,6 +443,7 @@ __wt_stat_refresh_connection_stats(void *stats_arg)
 	stats->async_flush.v = 0;
 	stats->async_full.v = 0;
 	stats->async_max_queue.v = 0;
+	stats->async_nowork.v = 0;
 	stats->async_op_alloc.v = 0;
 	stats->async_op_compact.v = 0;
 	stats->async_op_insert.v = 0;