WT-2945 Occasional hang running reconfigure fuzz test (#3098)

* Add a write barrier in front of __wt_cond_signal() to ensure the caller's flags meant to cause a thread to exit are seen by the thread.

* Make the LSM start/stop worker thread loops look the same.

5 files changed, 46 insertions, 43 deletions

diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c
index 0715a035807..3bcdfd7ecb1 100644
--- a/src/conn/conn_stat.c
+++ b/src/conn/conn_stat.c
@@ -469,12 +469,12 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp)
 }
 
 /*
- * __wt_statlog_log_one --
- *	Log a set of statistics into the configured statistics log. Requires
- *	that the server is not currently running.
+ * __statlog_on_close --
+ *	Log a set of statistics at close. Requires the server is not currently
+ * running.
  */
-int
-__wt_statlog_log_one(WT_SESSION_IMPL *session)
+static int
+__statlog_on_close(WT_SESSION_IMPL *session)
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
@@ -643,7 +643,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close)
 
 	/* Log a set of statistics on shutdown if configured. */
 	if (is_close)
-		WT_TRET(__wt_statlog_log_one(session));
+		WT_TRET(__statlog_on_close(session));
 
 	/* Discard all configuration information. */
 	WT_TRET(__stat_config_discard(session));
diff --git a/src/include/extern.h b/src/include/extern.h
index 79e6405e148..89b83ce6969 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -270,7 +270,6 @@ extern int __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_
 extern int __wt_connection_close(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern void __wt_conn_stat_init(WT_SESSION_IMPL *session);
-extern int __wt_statlog_log_one(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_sweep_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c
index 0a5f4fdd8b5..cbd83a5cd30 100644
--- a/src/lsm/lsm_manager.c
+++ b/src/lsm/lsm_manager.c
@@ -55,6 +55,7 @@ __lsm_general_worker_start(WT_SESSION_IMPL *session)
 	 * as many worker threads as are required to keep up with demand.
 	 */
 	WT_ASSERT(session, manager->lsm_workers > 0);
+	WT_ASSERT(session, manager->lsm_workers < manager->lsm_workers_max);
 	for (; manager->lsm_workers < manager->lsm_workers_max;
 	    manager->lsm_workers++) {
 		worker_args =
@@ -116,17 +117,18 @@ __lsm_stop_workers(WT_SESSION_IMPL *session)
 {
 	WT_LSM_MANAGER *manager;
 	WT_LSM_WORKER_ARGS *worker_args;
-	uint32_t i;
 
 	manager = &S2C(session)->lsm_manager;
 	/*
-	 * Start at the end of the list of threads and stop them until we
-	 * have the desired number.  We want to keep all active threads
-	 * packed at the front of the worker array.
+	 * Start at the end of the list of threads and stop them until we have
+	 * the desired number. We want to keep all active threads packed at the
+	 * front of the worker array.
 	 */
-	WT_ASSERT(session, manager->lsm_workers != 0);
-	for (i = manager->lsm_workers - 1; i >= manager->lsm_workers_max; i--) {
-		worker_args = &manager->lsm_worker_cookies[i];
+	WT_ASSERT(session, manager->lsm_workers > manager->lsm_workers_max);
+	for (; manager->lsm_workers > manager->lsm_workers_max;
+	    manager->lsm_workers--) {
+		worker_args =
+		    &manager->lsm_worker_cookies[manager->lsm_workers - 1];
 		/*
 		 * Clear this worker's flag so it stops.
 		 */
@@ -136,7 +138,6 @@ __lsm_stop_workers(WT_SESSION_IMPL *session)
 		worker_args->tid = 0;
 		worker_args->type = 0;
 		worker_args->flags = 0;
-		manager->lsm_workers--;
 		/*
 		 * We do not clear the session because they are allocated
 		 * statically when the connection was opened.
diff --git a/src/os_posix/os_mtx_cond.c b/src/os_posix/os_mtx_cond.c
index 842bb6eeec9..d4ee532a2af 100644
--- a/src/os_posix/os_mtx_cond.c
+++ b/src/os_posix/os_mtx_cond.c
@@ -103,27 +103,30 @@ void
 __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond)
 {
 	WT_DECL_RET;
-	bool locked;
-
-	locked = false;
 
 	__wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name);
 
-	/* Fast path if already signalled. */
-	if (cond->waiters == -1)
-		return;
+	/*
+	 * Our callers are often setting flags to cause a thread to exit. Add
+	 * a barrier to ensure the flags are seen by the threads.
+	 */
+	WT_WRITE_BARRIER();
 
-	if (cond->waiters > 0 || !__wt_atomic_casi32(&cond->waiters, 0, -1)) {
-		WT_ERR(pthread_mutex_lock(&cond->mtx));
-		locked = true;
-		WT_ERR(pthread_cond_broadcast(&cond->cond));
-	}
+	/*
+	 * Fast path if we are in (or can enter), a state where the next waiter
+	 * will return immediately as already signaled.
+	 */
+	if (cond->waiters == -1 ||
+	    (cond->waiters == 0 && __wt_atomic_casi32(&cond->waiters, 0, -1)))
+		return;
 
-err:	if (locked)
-		WT_TRET(pthread_mutex_unlock(&cond->mtx));
+	WT_ERR(pthread_mutex_lock(&cond->mtx));
+	ret = pthread_cond_broadcast(&cond->cond);
+	WT_TRET(pthread_mutex_unlock(&cond->mtx));
 	if (ret == 0)
 		return;
 
+err:
 	WT_PANIC_MSG(session, ret, "pthread_cond_broadcast: %s", cond->name);
 }
 
diff --git a/src/os_win/os_mtx_cond.c b/src/os_win/os_mtx_cond.c
index 41acec3ab4c..79c62ccd7f2 100644
--- a/src/os_win/os_mtx_cond.c
+++ b/src/os_win/os_mtx_cond.c
@@ -123,22 +123,23 @@ __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond)
 
 	__wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name);
 
-	/* Fast path if already signalled. */
-	if (cond->waiters == -1)
-		return;
-
-	if (cond->waiters > 0 || !__wt_atomic_casi32(&cond->waiters, 0, -1)) {
-		EnterCriticalSection(&cond->mtx);
-		locked = true;
-		WakeAllConditionVariable(&cond->cond);
-	}
+	/*
+	 * Our callers are often setting flags to cause a thread to exit. Add
+	 * a barrier to ensure the flags are seen by the threads.
+	 */
+	WT_WRITE_BARRIER();
 
-	if (locked)
-		LeaveCriticalSection(&cond->mtx);
-	if (ret == 0)
+	/*
+	 * Fast path if we are in (or can enter), a state where the next waiter
+	 * will return immediately as already signaled.
+	 */
+	if (cond->waiters == -1 ||
+	    (cond->waiters == 0 && __wt_atomic_casi32(&cond->waiters, 0, -1)))
 		return;
 
-	WT_PANIC_MSG(session, ret, "WakeAllConditionVariable: %s", cond->name);
+	EnterCriticalSection(&cond->mtx);
+	WakeAllConditionVariable(&cond->cond);
+	LeaveCriticalSection(&cond->mtx);
 }
 
 /*
@@ -149,7 +150,6 @@ int
 __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp)
 {
 	WT_CONDVAR *cond;
-	WT_DECL_RET;
 
 	cond = *condp;
 	if (cond == NULL)
@@ -159,5 +159,5 @@ __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp)
 	DeleteCriticalSection(&cond->mtx);
 	__wt_free(session, *condp);
 
-	return (ret);
+	return (0);
 }