WT-3155 Remove WT_CONN_SERVER_RUN flag (#3344)

Set WT_CONN_CLOSING earlier in the connection close process (before
calling the async close functions). This requires removing the assert
in btree handle open that close hasn't yet been called. Add a barrier
after setting the connection close flag to ensure the write is flushed.

LSM workers checked both the WT_CONN_SERVER_RUN and WT_LSM_WORKER_RUN
flags because the LSM destroy path (__lsm_manager_worker_shutdown),
didn't clear WT_LSM_WORKER_RUN flag. Add that clear, change __lsm_worker
to only check WT_LSM_WORKER_RUN.

Previously, the LSM manager checked the WT_CONN_SERVER_RUN flag in the
LSM destroy path and connection shutdown waited on the LSM manager to
stop and clear WT_CONN_SERVER_LSM. Flip that process: the LSM shutdown
path now clears WT_CONN_SERVER_LSM, and the LSM manager stops when it
sees WT_CONN_SERVER_LSM is cleared. The LSM manager sets a new flag,
WT_LSM_MANAGER_SHUTDOWN, when it's stopped, and the shutdown process
waits on that new flag.

Add memory barriers to the thread create and join functions. WiredTiger
typically sets (clears) state and expects threads to see the state and
start (stop). It simpler and safer if we imply a barrier in the thread
API.

* Rename WT_CONN_LOG_SERVER_RUN to WT_CONN_SERVER_LOG to match the other
server flags.

* Once the async and LSM servers have exited, assert no more files are
opened.

* Instead of using a barrier to ensure the worker run state isn't cached,
declare the structure field volatile. Use a stand-alone structure field
instead of a set of flags, it's a simpler "volatile" story.

* In one of two places, when shutting down worker threads, we signalled the
condition variable to wake the worker thread. For consistency, remove the
signal (we're only sleeping for 100th of a second, the wake isn't buying
us anything).

* Restore the assertion in __open_session() that we're not in the
"closing" path, returning an error is more dangerous, it might
cause a thread to panic, and then we have a panic racing with the
close.

* A wt_thread_t (POSIX pthread_t) is an opaque type, and can't be assigned
to 0 or tested against an integral value portably. Add a
bool WT_LSM_WORKER_ARGS.tid_set field instead of assigning or testing the
wt_thread_t.

We already have an __wt_lsm_start function, add a __wt_lsm_stop function
and move the setting/clearing of the WT_LSM_WORKER_ARGS.{running,tid_set}
fields into those functions so we ensure the ordering is correct.

3 files changed, 16 insertions, 9 deletions

diff --git a/src/include/extern.h b/src/include/extern.h
index 2759ac1dec3..47b4e03a7b7 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -458,6 +458,7 @@ extern int __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree)
 extern int __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
+extern int __wt_lsm_worker_stop(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_meta_checkpoint_last_name( WT_SESSION_IMPL *session, const char *fname, const char **namep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
diff --git a/src/include/flags.h b/src/include/flags.h
index c1fff920e3b..f26a45c68f5 100644
--- a/src/include/flags.h
+++ b/src/include/flags.h
@@ -6,19 +6,19 @@
 #define	WT_CONN_CACHE_POOL				0x00000001
 #define	WT_CONN_CKPT_SYNC				0x00000002
 #define	WT_CONN_CLOSING					0x00000004
-#define	WT_CONN_EVICTION_RUN				0x00000008
-#define	WT_CONN_IN_MEMORY				0x00000010
-#define	WT_CONN_LAS_OPEN				0x00000020
-#define	WT_CONN_LEAK_MEMORY				0x00000040
-#define	WT_CONN_LOG_SERVER_RUN				0x00000080
+#define	WT_CONN_CLOSING_NO_MORE_OPENS			0x00000008
+#define	WT_CONN_EVICTION_RUN				0x00000010
+#define	WT_CONN_IN_MEMORY				0x00000020
+#define	WT_CONN_LAS_OPEN				0x00000040
+#define	WT_CONN_LEAK_MEMORY				0x00000080
 #define	WT_CONN_LSM_MERGE				0x00000100
 #define	WT_CONN_PANIC					0x00000200
 #define	WT_CONN_READONLY				0x00000400
 #define	WT_CONN_RECOVERING				0x00000800
 #define	WT_CONN_SERVER_ASYNC				0x00001000
 #define	WT_CONN_SERVER_CHECKPOINT			0x00002000
-#define	WT_CONN_SERVER_LSM				0x00004000
-#define	WT_CONN_SERVER_RUN				0x00008000
+#define	WT_CONN_SERVER_LOG				0x00004000
+#define	WT_CONN_SERVER_LSM				0x00008000
 #define	WT_CONN_SERVER_STATISTICS			0x00010000
 #define	WT_CONN_SERVER_SWEEP				0x00020000
 #define	WT_CONN_WAS_BACKUP				0x00040000
diff --git a/src/include/lsm.h b/src/include/lsm.h
index 2bbb813bad2..e3f6897ef9d 100644
--- a/src/include/lsm.h
+++ b/src/include/lsm.h
@@ -23,11 +23,14 @@ struct __wt_lsm_worker_cookie {
 struct __wt_lsm_worker_args {
 	WT_SESSION_IMPL	*session;	/* Session */
 	WT_CONDVAR	*work_cond;	/* Owned by the manager */
+
 	wt_thread_t	tid;		/* Thread id */
+	bool		tid_set;	/* Thread id set */
+
 	u_int		id;		/* My manager slot id */
 	uint32_t	type;		/* Types of operations handled */
-#define	WT_LSM_WORKER_RUN	0x01
-	uint32_t	flags;		/* Worker flags */
+
+	volatile bool	running;	/* Worker is running */
 };
 
 /*
@@ -162,6 +165,9 @@ struct __wt_lsm_manager {
 #define	WT_LSM_MAX_WORKERS	20
 #define	WT_LSM_MIN_WORKERS	3
 	WT_LSM_WORKER_ARGS lsm_worker_cookies[WT_LSM_MAX_WORKERS];
+
+#define	WT_LSM_MANAGER_SHUTDOWN	0x01	/* Manager has shut down */
+	uint32_t flags;
 };
 
 /*